aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorudovichenko-r <rvu@ydb.tech>2023-08-28 17:11:12 +0300
committerudovichenko-r <rvu@ydb.tech>2023-08-28 17:49:09 +0300
commit4e949f902f6ad6f88dbd27508547fd04ccfc413a (patch)
tree20fb5159bf96e16d166d7366123752a558b88532
parent1e82f33abca18eecf6a334a3a20c81398765b1a2 (diff)
downloadydb-4e949f902f6ad6f88dbd27508547fd04ccfc413a.tar.gz
[yql] cleanup ya.makes (part 1)
YQL-16404
-rw-r--r--contrib/libs/cxxsupp/libcxx/include/semaphore188
-rw-r--r--contrib/libs/protobuf/python/ya.make9
-rw-r--r--ydb/library/yql/core/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/core/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/core/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/core/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/library/yql/core/arrow_kernels/ya.make4
-rw-r--r--ydb/library/yql/core/services/ya.make4
-rw-r--r--ydb/library/yql/core/spilling/CMakeLists.darwin-x86_64.txt39
-rw-r--r--ydb/library/yql/core/spilling/CMakeLists.linux-aarch64.txt40
-rw-r--r--ydb/library/yql/core/spilling/CMakeLists.linux-x86_64.txt40
-rw-r--r--ydb/library/yql/core/spilling/CMakeLists.txt17
-rw-r--r--ydb/library/yql/core/spilling/CMakeLists.windows-x86_64.txt39
-rw-r--r--ydb/library/yql/core/spilling/interface/spilling.h195
-rw-r--r--ydb/library/yql/core/spilling/namespace_cache.cpp824
-rw-r--r--ydb/library/yql/core/spilling/namespace_cache.h215
-rw-r--r--ydb/library/yql/core/spilling/namespaces_list.cpp167
-rw-r--r--ydb/library/yql/core/spilling/namespaces_list.h54
-rw-r--r--ydb/library/yql/core/spilling/spilling_imp.cpp245
-rw-r--r--ydb/library/yql/core/spilling/spilling_imp.h98
-rw-r--r--ydb/library/yql/core/spilling/storage/CMakeLists.darwin-x86_64.txt31
-rw-r--r--ydb/library/yql/core/spilling/storage/CMakeLists.linux-aarch64.txt32
-rw-r--r--ydb/library/yql/core/spilling/storage/CMakeLists.linux-x86_64.txt32
-rw-r--r--ydb/library/yql/core/spilling/storage/CMakeLists.txt17
-rw-r--r--ydb/library/yql/core/spilling/storage/CMakeLists.windows-x86_64.txt31
-rw-r--r--ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.darwin-x86_64.txt29
-rw-r--r--ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-aarch64.txt30
-rw-r--r--ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-x86_64.txt30
-rw-r--r--ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.txt17
-rw-r--r--ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.windows-x86_64.txt29
-rw-r--r--ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp260
-rw-r--r--ydb/library/yql/core/spilling/storage/file_storage/ya.make32
-rw-r--r--ydb/library/yql/core/spilling/storage/storage.cpp141
-rw-r--r--ydb/library/yql/core/spilling/storage/storage.h121
-rw-r--r--ydb/library/yql/core/spilling/storage/ut/CMakeLists.darwin-x86_64.txt78
-rw-r--r--ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-aarch64.txt81
-rw-r--r--ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-x86_64.txt83
-rw-r--r--ydb/library/yql/core/spilling/storage/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/core/spilling/storage/ut/CMakeLists.windows-x86_64.txt71
-rw-r--r--ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp104
-rw-r--r--ydb/library/yql/core/spilling/storage/ut/ya.make32
-rw-r--r--ydb/library/yql/core/spilling/storage/ya.make36
-rw-r--r--ydb/library/yql/core/spilling/ut/CMakeLists.darwin-x86_64.txt79
-rw-r--r--ydb/library/yql/core/spilling/ut/CMakeLists.linux-aarch64.txt82
-rw-r--r--ydb/library/yql/core/spilling/ut/CMakeLists.linux-x86_64.txt84
-rw-r--r--ydb/library/yql/core/spilling/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/core/spilling/ut/CMakeLists.windows-x86_64.txt72
-rw-r--r--ydb/library/yql/core/spilling/ut/spilling_ut.cpp368
-rw-r--r--ydb/library/yql/core/spilling/ut/ya.make35
-rw-r--r--ydb/library/yql/core/spilling/ya.make52
-rw-r--r--ydb/library/yql/core/url_lister/ya.make4
-rw-r--r--ydb/library/yql/core/url_preprocessing/ya.make4
-rw-r--r--ydb/library/yql/core/ya.make21
-rw-r--r--ydb/library/yql/minikql/arrow/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/arrow/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/minikql/arrow/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/arrow/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/arrow/ut/CMakeLists.darwin-x86_64.txt81
-rw-r--r--ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-aarch64.txt84
-rw-r--r--ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-x86_64.txt86
-rw-r--r--ydb/library/yql/minikql/arrow/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/arrow/ut/CMakeLists.windows-x86_64.txt74
-rw-r--r--ydb/library/yql/minikql/codegen/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/codegen/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/minikql/codegen/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/codegen/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/codegen/ut/CMakeLists.darwin-x86_64.txt157
-rw-r--r--ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-aarch64.txt160
-rw-r--r--ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-x86_64.txt162
-rw-r--r--ydb/library/yql/minikql/codegen/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/codegen/ut/CMakeLists.windows-x86_64.txt150
-rw-r--r--ydb/library/yql/minikql/comp_nodes/CMakeLists.txt2
-rw-r--r--ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.darwin-x86_64.txt157
-rw-r--r--ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-aarch64.txt158
-rw-r--r--ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-x86_64.txt158
-rw-r--r--ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.windows-x86_64.txt157
-rw-r--r--ydb/library/yql/minikql/comp_nodes/no_llvm/ya.make11
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.darwin-x86_64.txt124
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-aarch64.txt127
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-x86_64.txt129
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.windows-x86_64.txt117
-rwxr-xr-xydb/library/yql/minikql/comp_nodes/ut/build_no_codegen.sh4
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp67
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp227
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp179
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp737
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp254
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp487
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp1576
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp1119
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp4799
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp496
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp2329
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp393
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp1120
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp853
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp1207
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp38
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp2003
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp254
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp319
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp431
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp329
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp469
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp1150
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp172
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp305
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp626
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp171
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp271
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp143
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp523
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp276
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp549
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp527
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp316
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp474
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp1699
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp173
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp386
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp253
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp124
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp58
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp651
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/run_codegen.cmd9
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/ya.make83
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ya.make8
-rw-r--r--ydb/library/yql/minikql/computation/CMakeLists.txt2
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp124
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp2
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp380
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp126
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp863
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp837
-rw-r--r--ydb/library/yql/minikql/computation/mkql_validate_ut.cpp1153
-rw-r--r--ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp373
-rw-r--r--ydb/library/yql/minikql/computation/no_llvm/CMakeLists.darwin-x86_64.txt52
-rw-r--r--ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-aarch64.txt53
-rw-r--r--ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-x86_64.txt53
-rw-r--r--ydb/library/yql/minikql/computation/no_llvm/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/computation/no_llvm/CMakeLists.windows-x86_64.txt52
-rw-r--r--ydb/library/yql/minikql/computation/no_llvm/ya.make9
-rw-r--r--ydb/library/yql/minikql/computation/presort_ut.cpp649
-rw-r--r--ydb/library/yql/minikql/computation/ut/CMakeLists.darwin-x86_64.txt90
-rw-r--r--ydb/library/yql/minikql/computation/ut/CMakeLists.linux-aarch64.txt93
-rw-r--r--ydb/library/yql/minikql/computation/ut/CMakeLists.linux-x86_64.txt95
-rw-r--r--ydb/library/yql/minikql/computation/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/computation/ut/CMakeLists.windows-x86_64.txt83
-rw-r--r--ydb/library/yql/minikql/computation/ut/ya.make38
-rw-r--r--ydb/library/yql/minikql/computation/ya.make8
-rw-r--r--ydb/library/yql/minikql/dom/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/dom/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/minikql/dom/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/dom/ut/CMakeLists.darwin-x86_64.txt74
-rw-r--r--ydb/library/yql/minikql/dom/ut/CMakeLists.linux-aarch64.txt77
-rw-r--r--ydb/library/yql/minikql/dom/ut/CMakeLists.linux-x86_64.txt79
-rw-r--r--ydb/library/yql/minikql/dom/ut/CMakeLists.txt15
-rw-r--r--ydb/library/yql/minikql/dom/ut/ya.make3
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/CMakeLists.txt2
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp51
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.darwin-x86_64.txt69
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-aarch64.txt70
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-x86_64.txt70
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.windows-x86_64.txt69
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/no_llvm/ya.make12
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.darwin-x86_64.txt80
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-aarch64.txt83
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-x86_64.txt85
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.windows-x86_64.txt73
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/ut/ya.make28
-rw-r--r--ydb/library/yql/minikql/invoke_builtins/ya.make8
-rw-r--r--ydb/library/yql/minikql/jsonpath/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/jsonpath/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/minikql/jsonpath/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/jsonpath/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/library/yql/minikql/jsonpath/ut/CMakeLists.darwin-x86_64.txt84
-rw-r--r--ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-aarch64.txt87
-rw-r--r--ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-x86_64.txt89
-rw-r--r--ydb/library/yql/minikql/jsonpath/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/jsonpath/ut/CMakeLists.windows-x86_64.txt77
-rw-r--r--ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp2
-rw-r--r--ydb/library/yql/minikql/jsonpath/ya.make4
-rw-r--r--ydb/library/yql/minikql/perf/CMakeLists.txt2
-rw-r--r--ydb/library/yql/minikql/perf/block_groupby/CMakeLists.darwin-x86_64.txt39
-rw-r--r--ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-aarch64.txt42
-rw-r--r--ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-x86_64.txt44
-rw-r--r--ydb/library/yql/minikql/perf/block_groupby/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/perf/block_groupby/CMakeLists.windows-x86_64.txt32
-rw-r--r--ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp497
-rw-r--r--ydb/library/yql/minikql/perf/block_groupby/ya.make19
-rw-r--r--ydb/library/yql/minikql/perf/mprefetch/CMakeLists.darwin-x86_64.txt32
-rw-r--r--ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-aarch64.txt37
-rw-r--r--ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-x86_64.txt39
-rw-r--r--ydb/library/yql/minikql/perf/mprefetch/CMakeLists.txt17
-rw-r--r--ydb/library/yql/minikql/perf/mprefetch/CMakeLists.windows-x86_64.txt27
-rw-r--r--ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp108
-rw-r--r--ydb/library/yql/minikql/perf/mprefetch/ya.make17
-rw-r--r--ydb/library/yql/minikql/perf/ya.make10
-rw-r--r--ydb/library/yql/minikql/ya.make13
-rw-r--r--ydb/library/yql/parser/proto_ast/ya.make2
-rw-r--r--ydb/library/yql/providers/clickhouse/ya.make2
-rw-r--r--ydb/library/yql/providers/common/arrow/ya.make6
-rw-r--r--ydb/library/yql/providers/common/metrics/ya.make4
-rw-r--r--ydb/library/yql/providers/common/proto/python/ya.make2
-rw-r--r--ydb/library/yql/providers/common/schema/ya.make7
-rw-r--r--ydb/library/yql/providers/common/ya.make9
-rw-r--r--ydb/library/yql/providers/dq/provider/ya.make4
-rw-r--r--ydb/library/yql/providers/dq/worker_manager/ya.make4
-rw-r--r--ydb/library/yql/providers/dq/ya.make1
-rw-r--r--ydb/library/yql/providers/generic/ya.make2
-rw-r--r--ydb/library/yql/providers/s3/object_listers/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/providers/s3/object_listers/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.darwin-x86_64.txt67
-rw-r--r--ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-aarch64.txt70
-rw-r--r--ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-x86_64.txt72
-rw-r--r--ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.windows-x86_64.txt60
-rw-r--r--ydb/library/yql/providers/s3/ya.make1
-rw-r--r--ydb/library/yql/providers/stat/ya.make5
-rw-r--r--ydb/library/yql/providers/ya.make5
-rw-r--r--ydb/library/yql/providers/yt/codec/ya.make4
-rw-r--r--ydb/library/yql/providers/yt/comp_nodes/ya.make4
-rw-r--r--ydb/library/yql/providers/yt/lib/ya.make3
-rw-r--r--ydb/library/yql/providers/yt/ya.make5
-rw-r--r--ydb/library/yql/public/issue/ya.make4
-rw-r--r--ydb/library/yql/public/purecalc/examples/protobuf_pull_list/ya.make4
-rw-r--r--ydb/library/yql/public/purecalc/examples/ya.make2
-rw-r--r--ydb/library/yql/public/purecalc/io_specs/protobuf/ya.make4
-rw-r--r--ydb/library/yql/public/purecalc/ya.make8
-rw-r--r--ydb/library/yql/public/udf/service/ya.make5
-rw-r--r--ydb/library/yql/public/udf/tz/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/public/udf/tz/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/public/udf/tz/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/public/udf/tz/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/library/yql/public/udf/tz/ut/CMakeLists.darwin-x86_64.txt65
-rw-r--r--ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-aarch64.txt70
-rw-r--r--ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-x86_64.txt72
-rw-r--r--ydb/library/yql/public/udf/tz/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/public/udf/tz/ut/CMakeLists.windows-x86_64.txt60
-rw-r--r--ydb/library/yql/public/udf/tz/ya.make4
-rw-r--r--ydb/library/yql/public/udf/ya.make7
-rw-r--r--ydb/library/yql/public/ya.make8
-rw-r--r--ydb/library/yql/sql/v0/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/sql/v0/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/sql/v0/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/sql/v0/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/library/yql/sql/v0/ut/CMakeLists.darwin-x86_64.txt77
-rw-r--r--ydb/library/yql/sql/v0/ut/CMakeLists.linux-aarch64.txt80
-rw-r--r--ydb/library/yql/sql/v0/ut/CMakeLists.linux-x86_64.txt82
-rw-r--r--ydb/library/yql/sql/v0/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/sql/v0/ut/CMakeLists.windows-x86_64.txt70
-rw-r--r--ydb/library/yql/sql/v0/ya.make4
-rw-r--r--ydb/library/yql/sql/v1/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/sql/v1/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/sql/v1/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/sql/v1/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/library/yql/sql/v1/format/CMakeLists.darwin-x86_64.txt1
-rw-r--r--ydb/library/yql/sql/v1/format/CMakeLists.linux-aarch64.txt1
-rw-r--r--ydb/library/yql/sql/v1/format/CMakeLists.linux-x86_64.txt1
-rw-r--r--ydb/library/yql/sql/v1/format/CMakeLists.windows-x86_64.txt1
-rw-r--r--ydb/library/yql/sql/v1/format/ut/CMakeLists.darwin-x86_64.txt67
-rw-r--r--ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-aarch64.txt70
-rw-r--r--ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-x86_64.txt72
-rw-r--r--ydb/library/yql/sql/v1/format/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/sql/v1/format/ut/CMakeLists.windows-x86_64.txt60
-rw-r--r--ydb/library/yql/sql/v1/ut/CMakeLists.darwin-x86_64.txt79
-rw-r--r--ydb/library/yql/sql/v1/ut/CMakeLists.linux-aarch64.txt82
-rw-r--r--ydb/library/yql/sql/v1/ut/CMakeLists.linux-x86_64.txt84
-rw-r--r--ydb/library/yql/sql/v1/ut/CMakeLists.txt17
-rw-r--r--ydb/library/yql/sql/v1/ut/CMakeLists.windows-x86_64.txt72
-rw-r--r--ydb/library/yql/sql/v1/ya.make7
-rw-r--r--ydb/library/yql/sql/ya.make3
-rw-r--r--ydb/library/yql/tools/mrjob/ya.make4
-rw-r--r--ydb/library/yql/tools/ya.make1
-rw-r--r--ydb/library/yql/utils/backtrace/ya.make4
-rw-r--r--ydb/library/yql/utils/ya.make1
-rw-r--r--ydb/library/yql/ya.make15
283 files changed, 44288 insertions, 54 deletions
diff --git a/contrib/libs/cxxsupp/libcxx/include/semaphore b/contrib/libs/cxxsupp/libcxx/include/semaphore
new file mode 100644
index 00000000000..753d50f5126
--- /dev/null
+++ b/contrib/libs/cxxsupp/libcxx/include/semaphore
@@ -0,0 +1,188 @@
+// -*- C++ -*-
+//===----------------------------------------------------------------------===//
+//
+// Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
+// See https://llvm.org/LICENSE.txt for license information.
+// SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
+//
+//===----------------------------------------------------------------------===//
+
+#ifndef _LIBCPP_SEMAPHORE
+#define _LIBCPP_SEMAPHORE
+
+/*
+ semaphore synopsis
+
+namespace std {
+
+template<ptrdiff_t least_max_value = implementation-defined>
+class counting_semaphore
+{
+public:
+static constexpr ptrdiff_t max() noexcept;
+
+constexpr explicit counting_semaphore(ptrdiff_t desired);
+~counting_semaphore();
+
+counting_semaphore(const counting_semaphore&) = delete;
+counting_semaphore& operator=(const counting_semaphore&) = delete;
+
+void release(ptrdiff_t update = 1);
+void acquire();
+bool try_acquire() noexcept;
+template<class Rep, class Period>
+ bool try_acquire_for(const chrono::duration<Rep, Period>& rel_time);
+template<class Clock, class Duration>
+ bool try_acquire_until(const chrono::time_point<Clock, Duration>& abs_time);
+
+private:
+ptrdiff_t counter; // exposition only
+};
+
+using binary_semaphore = counting_semaphore<1>;
+
+}
+
+*/
+
+#include <__availability>
+#include <__config>
+#include <__thread/timed_backoff_policy.h>
+#include <__threading_support>
+#include <atomic>
+#include <version>
+
+#if !defined(_LIBCPP_HAS_NO_PRAGMA_SYSTEM_HEADER)
+# pragma GCC system_header
+#endif
+
+#ifdef _LIBCPP_HAS_NO_THREADS
+# error <semaphore> is not supported on this single threaded system
+#endif
+
+_LIBCPP_PUSH_MACROS
+#include <__undef_macros>
+
+#if _LIBCPP_STD_VER >= 14
+
+_LIBCPP_BEGIN_NAMESPACE_STD
+
+/*
+
+__atomic_semaphore_base is the general-case implementation.
+It is a typical Dijkstra semaphore algorithm over atomics, wait and notify
+functions. It avoids contention against users' own use of those facilities.
+
+*/
+
+class __atomic_semaphore_base
+{
+ __atomic_base<ptrdiff_t> __a;
+
+public:
+ _LIBCPP_INLINE_VISIBILITY
+ constexpr explicit __atomic_semaphore_base(ptrdiff_t __count) : __a(__count)
+ {
+ }
+ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
+ void release(ptrdiff_t __update = 1)
+ {
+ if(0 < __a.fetch_add(__update, memory_order_release))
+ ;
+ else if(__update > 1)
+ __a.notify_all();
+ else
+ __a.notify_one();
+ }
+ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
+ void acquire()
+ {
+ auto const __test_fn = [this]() -> bool {
+ auto __old = __a.load(memory_order_relaxed);
+ return (__old != 0) && __a.compare_exchange_strong(__old, __old - 1, memory_order_acquire, memory_order_relaxed);
+ };
+ __cxx_atomic_wait(&__a.__a_, __test_fn);
+ }
+ template <class Rep, class Period>
+ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
+ bool try_acquire_for(chrono::duration<Rep, Period> const& __rel_time)
+ {
+ if (__rel_time == chrono::duration<Rep, Period>::zero())
+ return try_acquire();
+ auto const __test_fn = [this]() { return try_acquire(); };
+ return __libcpp_thread_poll_with_backoff(__test_fn, __libcpp_timed_backoff_policy(), __rel_time);
+ }
+ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
+ bool try_acquire()
+ {
+ auto __old = __a.load(memory_order_acquire);
+ while (true) {
+ if (__old == 0)
+ return false;
+ if (__a.compare_exchange_strong(__old, __old - 1, memory_order_acquire, memory_order_relaxed))
+ return true;
+ }
+ }
+};
+
+#define _LIBCPP_SEMAPHORE_MAX (numeric_limits<ptrdiff_t>::max())
+
+template<ptrdiff_t __least_max_value = _LIBCPP_SEMAPHORE_MAX>
+class counting_semaphore
+{
+ __atomic_semaphore_base __semaphore;
+
+public:
+ static constexpr ptrdiff_t max() noexcept {
+ return __least_max_value;
+ }
+
+ _LIBCPP_INLINE_VISIBILITY
+ constexpr explicit counting_semaphore(ptrdiff_t __count) : __semaphore(__count) { }
+ ~counting_semaphore() = default;
+
+ counting_semaphore(const counting_semaphore&) = delete;
+ counting_semaphore& operator=(const counting_semaphore&) = delete;
+
+ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
+ void release(ptrdiff_t __update = 1)
+ {
+ __semaphore.release(__update);
+ }
+ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
+ void acquire()
+ {
+ __semaphore.acquire();
+ }
+ template<class Rep, class Period>
+ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
+ bool try_acquire_for(chrono::duration<Rep, Period> const& __rel_time)
+ {
+ return __semaphore.try_acquire_for(chrono::duration_cast<chrono::nanoseconds>(__rel_time));
+ }
+ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
+ bool try_acquire()
+ {
+ return __semaphore.try_acquire();
+ }
+ template <class Clock, class Duration>
+ _LIBCPP_AVAILABILITY_SYNC _LIBCPP_INLINE_VISIBILITY
+ bool try_acquire_until(chrono::time_point<Clock, Duration> const& __abs_time)
+ {
+ auto const current = Clock::now();
+ if (current >= __abs_time)
+ return try_acquire();
+ else
+ return try_acquire_for(__abs_time - current);
+ }
+};
+
+using binary_semaphore = counting_semaphore<1>;
+
+_LIBCPP_END_NAMESPACE_STD
+
+#endif // _LIBCPP_STD_VER >= 14
+
+_LIBCPP_POP_MACROS
+
+#endif //_LIBCPP_SEMAPHORE
diff --git a/contrib/libs/protobuf/python/ya.make b/contrib/libs/protobuf/python/ya.make
new file mode 100644
index 00000000000..9031af5cbba
--- /dev/null
+++ b/contrib/libs/protobuf/python/ya.make
@@ -0,0 +1,9 @@
+PACKAGE()
+
+WITHOUT_LICENSE_TEXTS()
+
+LICENSE(BSD-3-Clause)
+
+GENERATE_PY_PROTOS(contrib/libs/protobuf/src/google/protobuf/descriptor.proto)
+
+END()
diff --git a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt
index 1101ee7d42f..a29f25c4c65 100644
--- a/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/core/CMakeLists.darwin-x86_64.txt
@@ -18,6 +18,7 @@ add_subdirectory(file_storage)
add_subdirectory(issue)
add_subdirectory(peephole_opt)
add_subdirectory(services)
+add_subdirectory(spilling)
add_subdirectory(sql_types)
add_subdirectory(type_ann)
add_subdirectory(url_lister)
diff --git a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt
index e70610f312a..ad218bf3183 100644
--- a/ydb/library/yql/core/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/core/CMakeLists.linux-aarch64.txt
@@ -18,6 +18,7 @@ add_subdirectory(file_storage)
add_subdirectory(issue)
add_subdirectory(peephole_opt)
add_subdirectory(services)
+add_subdirectory(spilling)
add_subdirectory(sql_types)
add_subdirectory(type_ann)
add_subdirectory(url_lister)
diff --git a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt
index e70610f312a..ad218bf3183 100644
--- a/ydb/library/yql/core/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/core/CMakeLists.linux-x86_64.txt
@@ -18,6 +18,7 @@ add_subdirectory(file_storage)
add_subdirectory(issue)
add_subdirectory(peephole_opt)
add_subdirectory(services)
+add_subdirectory(spilling)
add_subdirectory(sql_types)
add_subdirectory(type_ann)
add_subdirectory(url_lister)
diff --git a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt
index 1101ee7d42f..a29f25c4c65 100644
--- a/ydb/library/yql/core/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/core/CMakeLists.windows-x86_64.txt
@@ -18,6 +18,7 @@ add_subdirectory(file_storage)
add_subdirectory(issue)
add_subdirectory(peephole_opt)
add_subdirectory(services)
+add_subdirectory(spilling)
add_subdirectory(sql_types)
add_subdirectory(type_ann)
add_subdirectory(url_lister)
diff --git a/ydb/library/yql/core/arrow_kernels/ya.make b/ydb/library/yql/core/arrow_kernels/ya.make
new file mode 100644
index 00000000000..9e858039fc6
--- /dev/null
+++ b/ydb/library/yql/core/arrow_kernels/ya.make
@@ -0,0 +1,4 @@
+RECURSE(
+ registry
+ request
+)
diff --git a/ydb/library/yql/core/services/ya.make b/ydb/library/yql/core/services/ya.make
index 45af1e24ff7..9a877d25514 100644
--- a/ydb/library/yql/core/services/ya.make
+++ b/ydb/library/yql/core/services/ya.make
@@ -36,3 +36,7 @@ PEERDIR(
YQL_LAST_ABI_VERSION()
END()
+
+RECURSE(
+ mounts
+)
diff --git a/ydb/library/yql/core/spilling/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/spilling/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..e222b22bb9b
--- /dev/null
+++ b/ydb/library/yql/core/spilling/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,39 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(storage)
+add_subdirectory(ut)
+
+add_library(yql-core-spilling)
+target_compile_options(yql-core-spilling PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(yql-core-spilling PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ ydb-library-binary_json
+ library-yql-utils
+ yql-utils-log
+ core-spilling-storage
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(yql-core-spilling PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/spilling_imp.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespaces_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespace_cache.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/spilling/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..de3eadabe42
--- /dev/null
+++ b/ydb/library/yql/core/spilling/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,40 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(storage)
+add_subdirectory(ut)
+
+add_library(yql-core-spilling)
+target_compile_options(yql-core-spilling PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(yql-core-spilling PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ ydb-library-binary_json
+ library-yql-utils
+ yql-utils-log
+ core-spilling-storage
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(yql-core-spilling PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/spilling_imp.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespaces_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespace_cache.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/spilling/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..de3eadabe42
--- /dev/null
+++ b/ydb/library/yql/core/spilling/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,40 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(storage)
+add_subdirectory(ut)
+
+add_library(yql-core-spilling)
+target_compile_options(yql-core-spilling PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(yql-core-spilling PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ ydb-library-binary_json
+ library-yql-utils
+ yql-utils-log
+ core-spilling-storage
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(yql-core-spilling PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/spilling_imp.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespaces_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespace_cache.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/CMakeLists.txt b/ydb/library/yql/core/spilling/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/core/spilling/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/spilling/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/spilling/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..e222b22bb9b
--- /dev/null
+++ b/ydb/library/yql/core/spilling/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,39 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(storage)
+add_subdirectory(ut)
+
+add_library(yql-core-spilling)
+target_compile_options(yql-core-spilling PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(yql-core-spilling PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ ydb-library-binary_json
+ library-yql-utils
+ yql-utils-log
+ core-spilling-storage
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(yql-core-spilling PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/spilling_imp.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespaces_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/namespace_cache.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/interface/spilling.h b/ydb/library/yql/core/spilling/interface/spilling.h
new file mode 100644
index 00000000000..1d32d3b110e
--- /dev/null
+++ b/ydb/library/yql/core/spilling/interface/spilling.h
@@ -0,0 +1,195 @@
+#pragma once
+
+#include <library/cpp/threading/future/async.h>
+#include <util/generic/buffer.h>
+
+namespace NYql {
+namespace NSpilling {
+
+struct TFileStorageConfig; // Configuration attributes to instantiate TempStorageProxy stored in filesystem folder
+struct TTempStorageExecutionPolicy; // Execution policy to work with temporary storage
+struct TTempObjectDesc; // Represents description of particular objects in temporary storage
+class ITempStorageProxy; // Proxy class to provide unified interface for all types of temporary storage
+struct TOperationResults; // Contains information about particular operation results
+
+// Factory method to create TempStorageProxy for usage. Error reasons are returned in TOperatonResults
+std::pair<THolder<ITempStorageProxy>, TOperationResults> CreateFileStorageProxy(const TFileStorageConfig& config, const TTempStorageExecutionPolicy& policy );
+
+struct TLoadOperationResults; // Contains information about load operation together with loaded data
+
+class ISession; // Session to manage temp objects lifecycle for particular process
+struct TSessionExecutionPolicy; // Execution policy for session to redefine default TTempStorageExecutionPolicy settings
+class IStream; // Class to save and load temp objects to storage in stream mode. Stream serializes objects order as it was saved, allows random
+ // access to stream objects by object index and provides consistency guarantee for all objects in stream when Close call succeed.
+
+
+// Iterator over temp storage objects
+class IObjectsIterator {
+public:
+
+ virtual bool Next(TTempObjectDesc& object) = 0; // Fills next temporary object description. Returns true if there are more objects
+ virtual ~IObjectsIterator() = default;
+};
+
+
+// Amount of data processed per session
+struct TSessionDataStat {
+ ui64 Provided = 0; // Total data provided by client to save in session (MB)
+ ui64 InMemory = 0; // Amount on data for session in memory (MB)
+ ui64 Spilled = 0; // Session data spilled to storage or deleted in memory (MB)
+ ui64 LoadedFromStorage = 0; // Amount of data loaded from storage (MB)
+ ui64 LoadedFromMemory = 0; // Session data processed in-memory without spilling (MB)
+};
+
+
+// Possible lifetime management options for objects stored in particular session
+enum class EObjectsLifetime {
+ DeleteAfterLoad = 0, // Object is deleted automatically from spilling storage after load operation. Object ownership is on the caller side after load call.
+ SharedForSession = 1, // Object can be loaded many times during session and object memory is managed by shared pointer. Pointer is shared between spilling Session and all callers for the object.
+ Persistent = 2, // Object is persistent and stored in spilling storage until deleted
+ };
+
+
+class ISession {
+public:
+ // Saves buf with size bytes to temporary storage. buf content should not be changed until operation completes.
+ // TOperationResults contains final results of buffer saving. buf is deleted after saving.
+ virtual NThreading::TFuture<TOperationResults> Save(const TString& objNamespace, const TString& name, TBuffer&& buf) = 0;
+
+ // Loads data from temporary storage to Buf in TLoadOperationResults.
+ virtual NThreading::TFuture<TLoadOperationResults> Load(const TString& objNamespace, const TString& name, EObjectsLifetime objLifetime = EObjectsLifetime::DeleteAfterLoad) = 0;
+
+ // Return current size of all temporary buffers to save. It should be used to control speed of Save operations from the client side
+ virtual TSessionDataStat GetSessionDataStat() = 0;
+
+ virtual TSessionExecutionPolicy ExecutionPolicy() = 0; // Returns current execution policy of ISession
+ virtual TOperationResults SetExecutionPolicy(const TSessionExecutionPolicy& policy) = 0; // Changes execution policy of ISession
+
+ // Opens stream to save or load data. Possible errors are returned in TOperationResults
+ virtual std::pair<THolder<IStream>, TOperationResults> OpenStream(const TString& objNamespace, const TString& streamName ) = 0;
+
+ virtual ~ISession() = default;
+};
+
+class IStream {
+public:
+ // Saves buf to stream. Buffer id in stream is returned in TOperationResults. Buffer id is a 0,1,2,3..(Size of stream - 1) sequence.
+ // Stream guarantees buffer id order the same as it was Save calls order.
+ virtual NThreading::TFuture<TOperationResults> Save(TBuffer&& buf) = 0;
+
+ // Return current number of objects stored in stream.
+ virtual ui64 Size() = 0;
+
+ // Loads data from stream with provided buffer id
+ virtual NThreading::TFuture<TLoadOperationResults> Load(ui64 bufferId = 0, EObjectsLifetime objLifetime = EObjectsLifetime::DeleteAfterLoad) = 0;
+
+ // Closes stream to ensure consistensy of all stream data buffers during subsequent load operations.
+ virtual NThreading::TFuture<TOperationResults> Close() = 0;
+
+ virtual ~IStream() = default;
+};
+
+
+
+
+
+class ITempStorageProxy {
+public:
+
+ // Creates new session to store and load temporary objects; Session manages lifecycle of all resources associated with the session.
+ // When session is deleted, all pending load operations are canceled, all resources associated with the session are freed
+ virtual THolder<ISession> CreateSession() = 0;
+
+ // Creates iterator to enumerate stored objects of interest. It works both for namespaces and objects enumeration.
+ // Objects are identified by namespace and name. If onlyValid = true, only valid objects are returned
+ virtual THolder<IObjectsIterator> CreateIterator( const TMaybe<TString>& objNamespace = TMaybe<TString>(),
+ const TMaybe<TString>& objName = TMaybe<TString>(),
+ bool onlyValid = true) = 0;
+
+
+ virtual TTempStorageExecutionPolicy ExecutionPolicy() = 0; // Returns current execution policy of ITempStorageProxy
+ virtual TOperationResults SetExecutionPolicy(const TTempStorageExecutionPolicy& policy) = 0; // Changes execution policy of ITempStorageProxy
+
+ // Deletes object with particular namespace and name. If name is empty, all objects from particular namespace are deleted.
+ virtual NThreading::TFuture<TOperationResults> Delete(const TString& objNamespace, const TMaybe<TString>& name) = 0;
+
+ virtual ~ITempStorageProxy() = default;
+
+};
+
+
+struct TTempStorageExecutionPolicy {
+ ui64 MaxStorageSize = 100; // Maximim size of temporary storage in GB. We are trying to keep total size of temporary objects withing this limit
+ ui64 RetentionPeriod = 24; // Default retention period in hours for temporary objects
+ ui64 DeleteOnClose = true; // When true, all temporary objects are deleted automatically when Session object is closed
+ bool CanClearNow = true; // If true, background deletion of obsolete objects is performed
+ ui64 MaxBandwidth = 10 * 1000; // ( Maximim bandwidth limitation for particular TempStorageProxy in MB/sec. Default is 10 GB/sec )
+ ui64 MaxCallsPerSecond = 100; // Maximim calls per second for particular TempStorageProxy
+ ui64 OneCallTimeout = 10; // Default timeout for one call to storage in seconds.
+ bool RetryCall = true; // If true, operation we are trying to complete operation using retry policy settings
+ ui64 RetryPeriod = 2; // Default period for retry calls in seconds
+ bool DoubleRetryPeriod = true; // Doubles retry period util it reaches MaxRetryPeriod;
+ ui64 MaxRetryPeriod = 60; // Maximim retry period in seconds
+ ui64 MaxNumberOfRetries = 10; // Maximim number of retries to complete operation.
+ ui64 MaxBuffersSize = 2000; // Total size of all process internals buffers in MB waiting either to load or store.
+ // When this limit exceeded, Save and Load operations starts returning BuffersAreFull error
+};
+
+struct TSessionExecutionPolicy {
+ ui64 DeleteOnClose = true; // When true, all temporary objects are deleted automatically when Session object is closed
+ ui64 MaxBuffersSize = 100; // Total size of session internals buffers in MB waiting either to load or store.
+ // When this limit exceeded, Save and Load operations starts returning BuffersAreFull error
+};
+
+// Possible reasons why object is considered invalid
+enum class EBadObjectReason {
+ NoBadReason = 0,
+ ChecksumInvalid = 1,
+ SizeInvalid = 2,
+ NoNamespace = 3,
+ NoObjectName = 4,
+ ReadError = 5
+ };
+
+struct TTempObjectDesc {
+ TString Namespace; // Namespace name. Empty for global namespace
+ TString Name; // Object name. Empty for namespaces. For stream objects it is a name of the stream
+ ui64 Size; // Size of the object in bytes. Total size of the objects in namespace in case of namespaces
+ ui64 Index; // Index of the object in the stream, starting from 0
+ bool IsValid; // True if object seems to be correct. Final consistency check is performed during full object load
+ bool IsFinal; // True if object is final in the stream
+ bool StreamValid; // True if stream is closed correctly
+ EBadObjectReason BadObjectReason; // Contains reason why object is considered invalid
+};
+
+// Possible statuses of operation
+enum class EOperationStatus {
+ Success = 0,
+ Failure = 1,
+ CannotOpenStorageProxy = 2,
+ WrongBufferId = 3,
+ ProxyTerminated = 4,
+ BuffersAreFull = 5,
+ NoObjectName = 6,
+ NewVersionDefined = 7,
+ ChecksumInvalid = 8
+};
+
+// Results of operation
+struct TOperationResults {
+ EOperationStatus Status; // Final status of operation
+ ui64 BufferId; // Buffer id for stream buffers sequence
+ TString ErrorString; // Contains error description for logs and following diagnostics
+};
+
+// Results of load operation
+struct TLoadOperationResults: public TOperationResults {
+ TAtomicSharedPtr<TBuffer> Buf; // Buffer with data after load operation.
+};
+
+struct TFileStorageConfig {
+ TString Path; // Storage path location
+};
+
+}
+} \ No newline at end of file
diff --git a/ydb/library/yql/core/spilling/namespace_cache.cpp b/ydb/library/yql/core/spilling/namespace_cache.cpp
new file mode 100644
index 00000000000..9faa9fbe9bb
--- /dev/null
+++ b/ydb/library/yql/core/spilling/namespace_cache.cpp
@@ -0,0 +1,824 @@
+#include "namespaces_list.h"
+#include "interface/spilling.h"
+#include "storage/storage.h"
+
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <map>
+
+#include <util/string/split.h>
+#include <contrib/libs/xxhash/xxhash.h>
+
+
+namespace NYql {
+namespace NSpilling {
+
+// Finds approximate position of val in array of N increasing values. Values cycle after max(ui32)
+// Returns true if val can be between start and end taking into account cycling.
+inline bool FindPos(ui32 start, ui32 end, ui32 val, ui32 nElements, ui32 & pos) {
+ if (!nElements)
+ return false;
+
+ if (start == end) {
+ if ( val == start ) {
+ pos = 0;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ if (start > end) {
+ ui32 len = (std::numeric_limits<ui32>::max() - start) + end;
+ if ((val >= start) || (val <= end)) {
+ pos = len / nElements;
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ if ((val >= start) && (val <= end)) {
+ pos = ( (val - start) * (nElements - 1 ) ) / (end - start);
+ return true;
+ } else {
+ return false;
+ }
+
+ return false;
+
+};
+
+// Returns file index for particular object id
+inline ui32 FindFileInd(ui32 objId) {
+
+ ui32 fileId = (objId) / (1<<16);
+ return fileId;
+
+}
+
+inline void TNamespaceCache::UpdateMinSessionId(ui32 sessionId, ui32 objId) {
+ auto sessionIt = Sessions_.find(sessionId);
+ if ( sessionIt == Sessions_.end() ) {
+ Sessions_[sessionId].MinObjId = objId;
+ } else {
+ if (sessionIt->second.MinObjId > objId) {
+ sessionIt->second.MinObjId = objId;
+ }
+ }
+}
+
+
+NThreading::TFuture<TOperationResults> TNamespaceCache::Save(const TString & objName, TBuffer && buf, ui32 sessionId, bool isStream ) {
+ NThreading::TFuture<TOperationResults> res;
+ TSaveTask bt;
+ bt.Name = objName;
+ bt.SessionId = sessionId;
+ bt.Buf = MakeAtomicShared<TBuffer>(std::move(buf));
+
+ bt.Promise = NThreading::NewPromise<TOperationResults>();
+
+ if ( !CurrSpillMetaFile_ ) {
+ NextNamespaceFile(true);
+ }
+
+
+ with_lock(ToSaveLock_) {
+ AdvanceObjId(bt.ObjId);
+ if ( !isStream ) {
+ ui32 lastObjId = 0;
+ auto search = LastObjs_.find(objName);
+ if (search != LastObjs_.end() ) {
+ lastObjId = search->second;
+ }
+ if ( lastObjId ) {
+ ui32 bytesDeleted = DeleteTaskFromSaveQueue(lastObjId);
+ SessionDataSpilled_[sessionId] += bytesDeleted;
+ }
+ LastObjs_[objName] = bt.ObjId;
+ } else {
+ auto search = LastStreamIds_.find(objName);
+ ui32 streamInd = 0;
+ if (search != LastStreamIds_.end() ) {
+ streamInd = search->second.size();
+ search->second.push_back(bt.ObjId);
+ } else {
+ streamInd = 0;
+ LastStreamIds_[objName].push_back(bt.ObjId);
+ }
+ bt.StreamBufId = streamInd;
+ bt.OpType = EOperationType::StreamBufAdd;
+ }
+ UpdateMinSessionId(sessionId, bt.ObjId);
+ SessionDataProvided_[sessionId] += bt.Buf->Size();
+ res = bt.Promise.GetFuture();
+ ToSave_.emplace_back(std::move(bt));
+ }
+ return res;
+}
+
+
+NThreading::TFuture<TLoadOperationResults> TNamespaceCache::Load(const TString& name, ui32 sessionId, EObjectsLifetime objLifetime, bool isStream, ui32 streamId ) {
+ NThreading::TFuture<TLoadOperationResults> res;
+ TLoadTask lt;
+ TLoadOperationResults lr;
+ lt.Name = name;
+ lt.Promise = NThreading::NewPromise<TLoadOperationResults>();
+ lt.Name = name;
+ lt.SessionId = sessionId;
+ lt.StreamBufId = streamId;
+ bool foundInLastObjs = false;
+ bool foundInSaveQueue = false;
+ with_lock(ToSaveLock_) {
+ if (!isStream) {
+ auto search = LastObjs_.find(name);
+ foundInLastObjs = (search != LastObjs_.end());
+ lt.ObjId = search->second;
+ } else {
+ auto search = LastStreamIds_.find(name);
+ if (search != LastStreamIds_.end()) {
+ if ( search->second.size() >= streamId ) {
+ foundInLastObjs = true;
+ lt.ObjId = search->second[streamId];
+ }
+ }
+ }
+ if ( foundInLastObjs ) {
+ UpdateMinSessionId(sessionId, lt.ObjId);
+ ui32 objPos = 0;
+ foundInSaveQueue = FindObjInSaveQueue(lt.ObjId, objPos);
+ if (foundInSaveQueue) {
+ TSaveTask & st = ToSave_[objPos];
+ if ( st.ProcessingStatus != EProcessingStatus::Deleted ) {
+ lr.Status = EOperationStatus::Success;
+ if ( st.ProcessingStatus == EProcessingStatus::Processing ) {
+ lr.Buf = st.Buf;
+ } else {
+ if (objLifetime == EObjectsLifetime::DeleteAfterLoad ) {
+ lr.Buf = st.Buf;
+ st.Buf = nullptr;
+ SessionDataSpilled_[st.SessionId] += lr.Buf->Size();
+ st.ProcessingStatus = EProcessingStatus::Deleted;
+ } else {
+ lr.Buf = st.Buf;
+ }
+
+ }
+ SessionDataLoadedFromMemory_[st.SessionId] += lr.Buf->Size();
+ lt.Promise.SetValue(std::move(lr));
+ } else {
+ lr.Status = EOperationStatus::NoObjectName;
+ lt.Promise.SetValue(std::move(lr));
+ }
+ }
+ } else {
+ lr.Status = EOperationStatus::NoObjectName;
+ lt.Promise.SetValue(std::move(lr));
+ }
+ }
+
+ res = lt.Promise.GetFuture();
+ if (!foundInSaveQueue && foundInLastObjs) {
+ with_lock(ToLoadLock_) {
+ ToLoad_.emplace_back(std::move(lt));
+ }
+ }
+
+ return res;
+
+}
+
+
+void TNamespaceCache::Dispatch() {
+
+ ProcessSaveQueue();
+ ProcessLoadQueue();
+ ProcessDeleteQueue();
+
+}
+
+
+TSessionDataStat TNamespaceCache::GetSessionStat(ui32 sessionId) {
+ TSessionDataStat res;
+ ui64 provided = 0;
+ ui64 spilled = 0;
+ ui64 loadedFromMemory = 0;
+ ui64 loadedFromStorage = 0;
+
+ with_lock(ToSaveLock_) {
+ auto it = SessionDataProvided_.find(sessionId);
+ if ( it != SessionDataProvided_.end()) {
+ provided = it->second;
+ res.Provided = provided / (1024 * 1024);
+ };
+ it = SessionDataSpilled_.find(sessionId);
+ if (it != SessionDataProvided_.end()) {
+ spilled = it->second;
+ res.Spilled = spilled / (1024 * 1024);
+ };
+ }
+
+ with_lock(ToLoadLock_) {
+ auto it = SessionDataLoadedFromMemory_.find(sessionId);
+ if (it != SessionDataLoadedFromMemory_.end()) {
+ loadedFromMemory = it->second;
+ res.LoadedFromMemory = loadedFromMemory / (1024 * 1024);
+ };
+
+ it = SessionDataLoadedFromStorage_.find(sessionId);
+ if (it != SessionDataLoadedFromStorage_.end()) {
+ loadedFromStorage = it->second;
+ res.LoadedFromStorage = loadedFromStorage / (1024 * 1024);
+ };
+
+
+ }
+
+ ui64 totalProcessed = spilled;
+ if ( totalProcessed > provided ) {
+ res.InMemory = 0;
+ } else {
+ res.InMemory = (provided - totalProcessed) / (1024 * 1024);
+ }
+
+ return res;
+}
+
+
+void TNamespaceCache::CloseSession(ui32 sessionId){
+ TBaseTask dt;
+ dt.OpType = EOperationType::SessionDelete;
+ dt.SessionId = sessionId;
+
+ with_lock(ToDeleteLock_) {
+ ToDelete_.emplace_back(dt);
+ }
+
+}
+
+void TNamespaceCache::GarbageCollection(){
+ std::set<ui32> filesIdToDelete;
+
+ ui32 minObjIdForSession = 0;
+ with_lock(ToSaveLock_) {
+ if (Sessions_.size() > 0) {
+ minObjIdForSession = Sessions_.begin()->second.MinObjId;
+ }
+ for (auto & s: Sessions_ ) {
+ if (s.second.MinObjId < minObjIdForSession ) {
+ minObjIdForSession = s.second.MinObjId;
+ }
+ }
+
+ for ( auto it = LastObjs_.begin(); it != LastObjs_.end(); ) {
+ if (it->second < minObjIdForSession ) {
+ it = LastObjs_.erase(it);
+ } else {
+ it++;
+ }
+ }
+
+ for ( auto it = LastStreamIds_.begin(); it != LastStreamIds_.end(); ) {
+ if ( it->second.size() > 0) {
+ if (it->second.front() < minObjIdForSession && it->second.back() < minObjIdForSession) {
+ it = LastStreamIds_.erase(it);
+ continue;
+ }
+ }
+ it++;
+ }
+
+
+ }
+
+
+ ui32 minFileId = FindFileInd(minObjIdForSession);
+ with_lock(FilesLock_) {
+ for (auto it = SpillMetaFiles_.begin(); it != SpillMetaFiles_.lower_bound(minFileId); it++ ) {
+ SpillFilesIdToDelete_.insert(it->first);
+ }
+ }
+
+
+
+ with_lock(FilesLock_) {
+ filesIdToDelete = SpillFilesIdToDelete_;
+
+ }
+
+ for (auto id: filesIdToDelete) {
+ TAtomicSharedPtr<TMetaFileAttributes> met;
+ with_lock(FilesLock_) {
+ InitMetaFile(id);
+ met = SpillMetaFiles_[id];
+ }
+ if ( met->DataFile->IsLocked()) {
+ met->DataFile->Delete();
+ }
+
+ if ( met->MetaFile->IsLocked()) {
+ met->MetaFile->Delete();
+ }
+ }
+
+ with_lock(FilesLock_) {
+ for (auto id: filesIdToDelete) {
+ SpillFilesIdToDelete_.erase(id);
+ SpillMetaFiles_.erase(id);
+
+ }
+
+ }
+
+}
+
+
+
+ui64 TNamespaceCache::StreamSize(const TString& name){
+ with_lock(ToSaveLock_) {
+ auto search = LastStreamIds_.find(name);
+ if (search != LastStreamIds_.end()) {
+ return search->second.size();
+ } else {
+ return 0;
+ }
+ }
+
+}
+
+
+void TNamespaceCache::NextNamespaceFile(bool openExisting) {
+ with_lock(FilesLock_) {
+ TVector<TString> files = StorageI_->GetNamespaceFiles(Name_);
+ ui32 maxInd = 0;
+ for (const auto & p: files) {
+ TVector<TString> splitted;
+ size_t tokenCount = Split(p, ".", splitted);
+ if (tokenCount == 4 && splitted[0] == "ydbspl" ) {
+ ui32 ind = 0;
+ if (TryFromString(splitted[1], ind)) {
+ if (SpillMetaFiles_.find(ind) == SpillMetaFiles_.end() ) {
+ SpillFilesIdToDelete_.insert(ind);
+ }
+ if (ind > maxInd) {
+ maxInd = ind;
+ }
+ }
+ }
+ }
+
+
+ if (maxInd > std::numeric_limits<ui32>::max() / 2 ) {
+ maxInd = 1;
+ }
+
+ SpillFilesIdToDelete_.erase(maxInd);
+
+ TTempStorageExecutionPolicy pol;
+ ui32 retryCount = 0;
+ for (; retryCount < pol.MaxNumberOfRetries; retryCount++ ) {
+ THolder<ISpillFile> metFile = StorageI_->CreateSpillFile(Name_, TString("ydbspl.") + std::to_string(maxInd) + TString(".0.met"), MetaFileIncreaseStep);
+ if (metFile->IsLocked()) {
+ CurrSpillFileId_ = maxInd;
+ CurrSpillMetaFile_ = std::move(metFile);
+ THolder<ISpillFile> datFile = StorageI_->CreateSpillFile(Name_, TString("ydbspl.") + std::to_string(maxInd) + TString(".0.dat"), DataFileIncreaseStep);
+ CurrSpillDataFile_ = std::move(datFile);
+ NextObjId_ = CurrSpillFileId_ * (1<<16) + 1;
+ TAtomicSharedPtr<TMetaFileAttributes> res = MakeAtomicShared<TMetaFileAttributes>();
+ res->MetaFile = CurrSpillMetaFile_;
+ res->DataFile = CurrSpillDataFile_;
+ res->Id = CurrSpillFileId_;
+ res->FirstObjId = NextObjId_;
+ res->MetaRecords = MakeAtomicShared<std::map<ui32, TSpillMetaRecord>>();
+ SpillMetaFiles_[CurrSpillFileId_] = res;
+ break;
+ } else {
+ maxInd++;
+ }
+ }
+
+ }
+
+
+}
+
+inline void TNamespaceCache::AdvanceObjId(ui32 & objId) {
+ if (NextObjId_ == std::numeric_limits<ui32>::max() ) {
+ NextObjId_ = 1;
+ }
+ objId = ++NextObjId_;
+}
+
+
+inline bool TNamespaceCache::FindObjInSaveQueue(ui32 objId, ui32& pos){
+ ui32 size = ToSave_.size();
+ if ( size == 0)
+ return false;
+ ui32 firstId = ToSave_.front().ObjId;
+ ui32 lastId = ToSave_.back().ObjId;
+ ui32 approximatePos = 0;
+ bool found = FindPos(firstId, lastId, objId, size, approximatePos );
+ if (!found)
+ return false;
+
+ pos = approximatePos;
+ auto it = ToSave_.begin() + pos;
+ ui32 toSaveId = it->ObjId;
+
+ if (toSaveId == objId) {
+ return true;
+ }
+
+ if (toSaveId < objId) {
+ for( ; it != ToSave_.end(); it++) {
+ toSaveId = it->ObjId;
+ if (toSaveId == objId) {
+ pos = (it - ToSave_.begin());
+ return true;
+ }
+ }
+ return false;
+ }
+
+ if (toSaveId > objId) {
+ while(pos > 0) {
+ --pos;
+ if (ToSave_[pos].ObjId == objId) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ return false;
+}
+
+
+ui32 TNamespaceCache::DeleteTaskFromSaveQueue(ui32 objId, EProcessingStatus reason) {
+ ui32 pos;
+ ui32 bytesDeleted = 0;
+ bool found = FindObjInSaveQueue(objId, pos);
+ if ( found && ToSave_[pos].ProcessingStatus != EProcessingStatus::Processing) {
+ ToSave_[pos].ProcessingStatus = reason;
+ bytesDeleted = ToSave_[pos].Buf->Size();
+ ToSave_[pos].Buf = nullptr;
+ }
+ return bytesDeleted;
+
+}
+
+bool TNamespaceCache::FindNextTaskInSaveQueue(ui32& taskPos) {
+
+ bool res = false;
+ for ( taskPos = 0; taskPos < ToSave_.size(); taskPos++) {
+ TSaveTask& saveTask = ToSave_[taskPos];
+ if ( saveTask.ProcessingStatus == EProcessingStatus::Added ) {
+ if (saveTask.OpType == EOperationType::Add ) {
+ TString& name = saveTask.Name;
+ auto search = LastObjs_.find(name);
+ ui32 foundObjId = 0;
+ if ( search != LastObjs_.end() ) {
+ foundObjId = search->second;
+ }
+
+ if (foundObjId <= saveTask.ObjId ) {
+ saveTask.ProcessingStatus = EProcessingStatus::Processing;
+ res = true;
+ break;
+ } else {
+ saveTask.ProcessingStatus = EProcessingStatus::Deleted;
+ }
+ }
+
+ if (saveTask.OpType == EOperationType::StreamBufAdd ) {
+ saveTask.ProcessingStatus = EProcessingStatus::Processing;
+ res = true;
+ break;
+
+ }
+
+ }
+
+ }
+ return res;
+}
+
+bool TNamespaceCache::FindNextTaskInLoadQueue(ui32& taskPos) {
+
+ bool res = false;
+ for ( taskPos = 0; taskPos < ToLoad_.size(); taskPos++) {
+ TLoadTask& loadTask = ToLoad_[taskPos];
+ if ( loadTask.ProcessingStatus == EProcessingStatus::Added ) {
+ if (loadTask.OpType == EOperationType::Add ) {
+ TString& name = loadTask.Name;
+ auto search = LastObjs_.find(name);
+ ui32 foundObjId = 0;
+ if ( search != LastObjs_.end() ) {
+ foundObjId = search->second;
+ }
+
+ if (foundObjId > loadTask.ObjId ) {
+ loadTask.ObjId = foundObjId;
+ }
+ loadTask.ProcessingStatus = EProcessingStatus::Processing;
+ res = true;
+ break;
+ }
+ }
+ }
+ return res;
+}
+
+bool TNamespaceCache::FindNextTaskInDeleteQueue(ui32& taskPos) {
+
+ bool res = false;
+ for ( taskPos = 0; taskPos < ToDelete_.size(); taskPos++) {
+ TBaseTask& deleteTask = ToDelete_[taskPos];
+ if ( deleteTask.ProcessingStatus == EProcessingStatus::Added ) {
+ deleteTask.ProcessingStatus = EProcessingStatus::Processing;
+ res = true;
+ break;
+ }
+ }
+ return res;
+}
+
+void TNamespaceCache::ChangeLastObjId(TString& objName, ui32 prevObjId, ui32 nextObjId) {
+ auto search = LastObjs_.find(objName);
+ if ( search != LastObjs_.end() ) {
+ if (search->second <= prevObjId) {
+ search->second = nextObjId;
+ }
+ }
+
+}
+
+void TNamespaceCache::ProcessSaveQueue() {
+
+ ui32 taskPos = 0;
+ bool saveTaskFound = false;
+ bool enoughSpaceToWrite = false;
+
+ try {
+ ToSaveLock_.lock();
+
+ RemoveCompletedFromSaveQueue();
+
+ saveTaskFound = FindNextTaskInSaveQueue(taskPos);
+
+ if ( saveTaskFound ) {
+
+ TAtomicSharedPtr<ISpillFile> currSpillMetaFile = CurrSpillMetaFile_;
+ TAtomicSharedPtr<ISpillFile> currSpillDataFile = CurrSpillDataFile_;
+
+ TSaveTask& saveTask = ToSave_[taskPos];
+ ui32 size = saveTask.Buf->Size();
+ ui64 offset = currSpillDataFile->Reserve(size);
+ ui64 total = offset + size;
+ char * data = saveTask.Buf->Data();
+ ui32 taskObjId = saveTask.ObjId;
+ ui32 prevObjId = taskObjId;
+ TSpillMetaRecord mr{EOperationType::Add, saveTask.Name, offset, taskObjId, size, 0 };
+ ui32 metaSize = mr.Size();
+ ui64 metaOffset = currSpillMetaFile->Reserve(metaSize);
+
+ if (total >= std::numeric_limits<ui32>::max() ) {
+ saveTask.ProcessingStatus = EProcessingStatus::Added;
+ NextNamespaceFile(false);
+ ToSaveLock_.unlock();
+ } else {
+ enoughSpaceToWrite = true;
+ saveTask.ProcessingStatus = EProcessingStatus::Processing;
+ bool changeObjId = ( !(taskObjId > CurrSpillFileId_ * (1<<16) && (taskObjId < (CurrSpillFileId_ + 1) * (1<<16)) ));
+ if (changeObjId) {
+ AdvanceObjId(taskObjId);
+ mr.SetObjId(taskObjId);
+ }
+ ToSaveLock_.unlock();
+ XXH32_hash_t hash = XXH32( data, size, 0);
+ mr.SetDataHash(hash);
+
+
+ TBuffer mrBuf;
+ mr.Pack(mrBuf);
+
+ currSpillDataFile->Write(offset, data, size);
+ currSpillMetaFile->Write(metaOffset, mrBuf.Data(), metaSize);
+
+
+ ToSaveLock_.lock();
+ saveTask.ProcessingStatus = EProcessingStatus::Deleted;
+ if (changeObjId) {
+ ChangeLastObjId(saveTask.Name, prevObjId, taskObjId);
+ }
+ SessionDataSpilled_[saveTask.SessionId] += size;
+ ToSaveLock_.unlock();
+
+ with_lock(FilesLock_) {
+ auto & metaAttributes = SpillMetaFiles_[CurrSpillFileId_];
+ metaAttributes->MetaRecords->insert({taskObjId, mr});
+ }
+ }
+
+ } else {
+ ToSaveLock_.unlock();
+ }
+
+
+ } catch(...) {
+ ToSaveLock_.unlock();
+ throw;
+ }
+
+}
+
+void TNamespaceCache::ProcessLoadQueue() {
+
+ ui32 taskPos = 0;
+ bool loadTaskFound = false;
+
+ with_lock(ToLoadLock_) {
+ RemoveCompletedFromLoadQueue();
+ loadTaskFound = FindNextTaskInLoadQueue(taskPos);
+ }
+
+ if (!loadTaskFound)
+ return;
+
+ TLoadTask& loadTask = ToLoad_[taskPos];
+ TLoadOperationResults lr;
+ ui32 fileInd = FindFileInd(loadTask.ObjId);
+ TSpillMetaRecord mr;
+ TAtomicSharedPtr<TMetaFileAttributes> fileMet;
+ bool found = false;
+
+ with_lock(FilesLock_) {
+ found = FindMetaRecordInFile(fileInd, loadTask.ObjId, mr, fileMet );
+ }
+
+
+ if (!found)
+ return;
+
+ TBuffer buf(mr.DataSize());
+ fileMet->DataFile->Read(mr.Offset(), buf.Data(), mr.DataSize());
+ XXH32_hash_t hash = XXH32( buf.Data(), mr.DataSize(), 0);
+ lr.Buf = MakeAtomicShared<TBuffer>(std::move(buf));
+ lr.Status = EOperationStatus::Success;
+
+ if ( hash != mr.DataHash() ) {
+ lr.Status = EOperationStatus::ChecksumInvalid;
+ YQL_LOG(ERROR) << "Wrong hash!!!: " << "Buf hash: " << hash << " Meta hash: " << mr.DataHash() << Endl;
+ }
+
+ SessionDataLoadedFromStorage_[loadTask.SessionId] += mr.DataSize();
+
+
+ loadTask.Promise.SetValue(std::move(lr));
+
+
+}
+
+void TNamespaceCache::ProcessDeleteQueue() {
+
+ ui32 taskPos = 0;
+ bool deleteTaskFound = false;
+
+ with_lock(ToDeleteLock_) {
+ RemoveCompletedFromDeleteQueue();
+ deleteTaskFound = FindNextTaskInDeleteQueue(taskPos);
+ }
+
+ if (!deleteTaskFound)
+ return;
+
+ TBaseTask& deleteTask = ToDelete_[taskPos];
+ ui32 sessionId = deleteTask.SessionId;
+
+ if (deleteTask.OpType == EOperationType::SessionDelete ) {
+ with_lock(ToSaveLock_) {
+ Sessions_.erase(sessionId);
+ for (auto& t : ToSave_ ) {
+ if ( t.SessionId == sessionId ) {
+ t.ProcessingStatus = EProcessingStatus::Deleted;
+ t.Buf = nullptr;
+ }
+ }
+ }
+
+ with_lock(ToLoadLock_) {
+ for (auto& t : ToLoad_ ) {
+ if ( t.SessionId == sessionId ) {
+ t.ProcessingStatus = EProcessingStatus::Deleted;
+ t.Buf = nullptr;
+ }
+ }
+ }
+ }
+
+
+
+}
+
+
+void TNamespaceCache::RemoveCompletedFromSaveQueue() {
+
+ for (auto it = ToSave_.begin(); it != ToSave_.end(); ) {
+ if (it->ProcessingStatus == EProcessingStatus::Deleted) {
+ ui64 sessionId = it->SessionId;
+ it = ToSave_.erase(it);
+ } else {
+ break;
+ }
+ }
+
+}
+
+void TNamespaceCache::RemoveCompletedFromLoadQueue() {
+
+ for (auto it = ToLoad_.begin(); it != ToLoad_.end(); ) {
+ if (it->ProcessingStatus == EProcessingStatus::Deleted) {
+ ui64 sessionId = it->SessionId;
+ it = ToLoad_.erase(it);
+ } else {
+ break;
+ }
+ }
+
+}
+
+
+void TNamespaceCache::RemoveCompletedFromDeleteQueue() {
+
+ for (auto it = ToDelete_.begin(); it != ToDelete_.end(); ) {
+ if (it->ProcessingStatus == EProcessingStatus::Deleted) {
+ ui64 sessionId = it->SessionId;
+ it = ToDelete_.erase(it);
+ } else {
+ break;
+ }
+ }
+
+}
+
+
+bool TNamespaceCache::FindMetaRecordInFile(ui32 fileId, ui32 objId, TSpillMetaRecord& mr, TAtomicSharedPtr<TMetaFileAttributes>& fileMet) {
+ bool res = false;
+ TAtomicSharedPtr<TMetaFileAttributes> met = InitMetaFile(fileId);
+ auto it = met->MetaRecords->find(objId);
+ if (it != met->MetaRecords->end() ) {
+ res = true;
+ mr = it->second;
+ fileMet = met;
+ }
+
+ return res;
+}
+
+TAtomicSharedPtr< std::vector<TSpillMetaRecord> > ReadAllRecordsFromMetaFile( TAtomicSharedPtr<ISpillFile> file) {
+ TAtomicSharedPtr< std::vector<TSpillMetaRecord> > res = MakeAtomicShared<std::vector<TSpillMetaRecord>>();
+ file->Seek(0);
+ TBuffer readbuf(ReadBufSize);
+ i32 readRes = file->Read(0, readbuf.Data(), ReadBufSize);
+ TSpillMetaRecord mr;
+ mr.Unpack(readbuf);
+ return res;
+}
+
+
+ TAtomicSharedPtr<TMetaFileAttributes> TNamespaceCache::InitMetaFile(ui32 fileId){
+ TAtomicSharedPtr<TMetaFileAttributes> res;
+
+ auto found = SpillMetaFiles_.find(fileId);
+ if (found != SpillMetaFiles_.end()) {
+ res = found->second;
+ } else {
+ THolder<ISpillFile> metFile = StorageI_->CreateSpillFile(Name_, TString("ydbspl.") + std::to_string(fileId) + TString(".0.met"), MetaFileIncreaseStep);
+ THolder<ISpillFile> datFile = StorageI_->CreateSpillFile(Name_, TString("ydbspl.") + std::to_string(fileId) + TString(".0.dat"), DataFileIncreaseStep);
+ res = MakeAtomicShared<TMetaFileAttributes>();
+ res->MetaFile = std::move( metFile );
+ res->DataFile = std::move(datFile);
+ SpillMetaFiles_[fileId] = res;
+ }
+
+
+
+ return res;
+}
+
+
+TAtomicSharedPtr<ISpillFile> FindSpillFile(ui32 objId, EFileType fileType = EFileType::Meta) {
+ TAtomicSharedPtr<ISpillFile> res;
+ return res;
+}
+
+TNamespaceCache::TNamespaceCache(const TString& name, ui32 id, TAtomicSharedPtr<ISpillStorage> storageI) :
+ Name_(name),
+ Id_(id),
+ StorageI_(storageI) {
+
+ };
+
+
+}
+}
diff --git a/ydb/library/yql/core/spilling/namespace_cache.h b/ydb/library/yql/core/spilling/namespace_cache.h
new file mode 100644
index 00000000000..2e648869d89
--- /dev/null
+++ b/ydb/library/yql/core/spilling/namespace_cache.h
@@ -0,0 +1,215 @@
+#include "interface/spilling.h"
+#include "storage/storage.h"
+
+#include <map>
+#include <deque>
+#include <set>
+#include <unordered_map>
+
+
+namespace NYql {
+namespace NSpilling {
+
+
+
+const ui32 MetaFileIncreaseStep = 10000; // 10 KB increase in meta file size
+const ui32 DataFileIncreaseStep = 100000000; // 100 MB increase in data file size
+const ui32 ReadBufSize = 4 * 1024 * 1024; // 4 MB read buffer size
+
+// Defines possible processing states of the task
+enum class EProcessingStatus {
+ Added = 1, // Task added to queue
+ Processing = 2, // Task is processing
+ Completed = 3, // Task is completed
+ NeedRetry = 4, // Scheduled for retry
+ Failed = 5, // Task is failed
+ Deleted = 6, // Task is deleted, no valid internal state
+ NewVersion = 7 // New version of object is available
+};
+
+
+// Base task for namespace operations
+struct TBaseTask {
+ EOperationType OpType = EOperationType::Add; // Type of operation
+ ui32 ObjId; // Identifier of the object
+ ui32 SessionId; // Identifier of the session for this task
+ TString Name; // Name of the object or stream
+ ui32 StreamBufId; // Id of the object inside the stream (0, 1, 2, ...)
+ TAtomicSharedPtr<TBuffer> Buf; // Data to save or load
+ EProcessingStatus ProcessingStatus = EProcessingStatus::Added; // Contains current status of task processing stage
+};
+
+
+// Task for save operations
+struct TSaveTask : public TBaseTask {
+ NThreading::TPromise<TOperationResults> Promise; // Promise to report operation results
+};
+
+
+// Task for load operations
+struct TLoadTask : public TBaseTask {
+ NThreading::TPromise<TLoadOperationResults> Promise; // Promise to report load operation results
+};
+
+// Defines possible retention period of file. All temporary files are deleted automacially on session close.
+// Persistent files store objects as defined in object retention period.
+enum class EFileLifespan {
+ Temporary = 0,
+ Persistent = 1
+};
+
+// Defines possible types of spill files. Meta files contain small index of objects stored in data files
+enum class EFileType {
+ Meta = 0,
+ Data = 1
+};
+
+// Namespace file description
+struct TNamespaceFileDesc {
+ TString Name; // Full name of the file
+ EFileType Type = EFileType::Meta; // Type of the file
+ EFileLifespan Lifespan = EFileLifespan::Temporary; // Lifespan of the spill file. All unlocked temporary files are deleted automatically
+ ui32 Id = 1; // Serial id of the file in the namespace. Id of the file defines interval of Object Ids, stored in file
+ ui32 Version = 1; // Version of the file in the namespace for the same id.
+
+};
+
+// Session info stored in the namespace cache
+struct TSessionDesc {
+ ui32 MinObjId; // Mininum object id associated with session
+};
+
+// Attributes of meta file to be used
+struct TMetaFileAttributes {
+ TAdaptiveLock Lock; // Lock to serialize access to file attributes
+ bool Valid; // True if namespace meta file exists and valid
+ TString Name; // Name of the file
+ ui32 Id = 0; // Id of the file in the namespace
+ TAtomicSharedPtr<ISpillFile> MetaFile; // File handle of the meta file
+ TAtomicSharedPtr<ISpillFile> DataFile; // File handle of the data file
+ ui32 StartPos = 0; // Start valid position in file
+ ui32 EndPos = 0; // End of valid position in file
+ ui32 FirstObjId = 0; // First object id in file
+ ui32 LastObjId = 0; // Last object id in file
+ TAtomicSharedPtr< std::map<ui32, TSpillMetaRecord> > MetaRecords; // Cached meta records of the file
+};
+
+
+
+// Class to represent internal cache for particular namespace
+class TNamespaceCache {
+public:
+ // Adds required spilling data to save queue
+ NThreading::TFuture<TOperationResults> Save(const TString& objName, TBuffer && buf, ui32 sessionId, bool isStream = false );
+
+ // Loads data from namespace cache or long term storage
+ NThreading::TFuture<TLoadOperationResults> Load(const TString& name, ui32 sessionId, EObjectsLifetime objLifetime = EObjectsLifetime::DeleteAfterLoad, bool isStream = false, ui32 streamId = 0 );
+
+ // Dispatches worker thread to process queues
+ void Dispatch();
+
+ // Return data statistics for particular session
+ TSessionDataStat GetSessionStat(ui32 sessionId);
+
+ // Closes session and clears all resources
+ void CloseSession(ui32 sessionId);
+
+ // Makes garbage collection for files stored in namespace
+ void GarbageCollection();
+
+ // Returns current number of objects stored in particular stream name
+ ui64 StreamSize(const TString& name);
+
+ // name - name of the namespace, id - internal id, storageI - storage interface to save Namespace Cache state
+ TNamespaceCache(const TString& name, ui32 id, TAtomicSharedPtr<ISpillStorage> storageI);
+private:
+
+ // Creates next namespace file. If openExisting = true, last namespace file is reused
+ void NextNamespaceFile(bool openExisting = true);
+
+ // Advances next object id, taking into account cyclic requirement
+ inline void AdvanceObjId(ui32& objId);
+
+ // Finds object with ObjId in Save queue. Returns true if found and object position in queue
+ inline bool FindObjInSaveQueue(ui32 objId, ui32& pos);
+
+ // Updates mininum object id for particular session if required
+ inline void UpdateMinSessionId(ui32 sessionId, ui32 objId);
+
+ // Deletes task from save queue. Returns bytes deleted in save task buffer.
+ ui32 DeleteTaskFromSaveQueue(ui32 objId, EProcessingStatus reason = EProcessingStatus::Deleted);
+
+ // Changes object id for the object
+ void ChangeLastObjId(TString& objName, ui32 prevObjId, ui32 nextObjId);
+
+ // Processes save queue
+ void ProcessSaveQueue();
+
+ // Processes load queue
+ void ProcessLoadQueue();
+
+ // Processes delete queue
+ void ProcessDeleteQueue();
+
+ // Finds next valid save task in save queue and returns true is task found. Task position in queue is returned in taskPos.
+ bool FindNextTaskInSaveQueue(ui32& taskPos);
+
+ // Finds next valid task in load queue and returns true is task found. Task position in queue is returned in taskPos.
+ bool FindNextTaskInLoadQueue(ui32& taskPos);
+
+ // Finds next valid task in delete queue and returns true is task found. Task position in queue is returned in taskPos.
+ bool FindNextTaskInDeleteQueue(ui32& taskPos);
+
+ // Removes all completed tasks from beginning of ToSave_ queue
+ void RemoveCompletedFromSaveQueue();
+
+ // Removes all completed tasks from beginning of ToLoad_ queue
+ void RemoveCompletedFromLoadQueue();
+
+ // Removes all completed tasks from beginning of ToDelete_ queue
+ void RemoveCompletedFromDeleteQueue();
+
+ // Returns true if object is found in meta file fileId.
+ // Meta record for particular object is returned in mr argument. File meta information is returned in fileMet record.
+ bool FindMetaRecordInFile(ui32 fileId, ui32 objId, TSpillMetaRecord& mr, TAtomicSharedPtr<TMetaFileAttributes>& fileMet );
+
+ // Inits meta file with file id
+ TAtomicSharedPtr<TMetaFileAttributes> InitMetaFile(ui32 fileId);
+
+ // Marks obj id as deleted
+ inline void SetDeletedBit(ui32 objId);
+
+ // Returns required file handle for particular object id
+ TAtomicSharedPtr<ISpillFile> FindSpillFile(ui32 objId, EFileType fileType = EFileType::Meta);
+
+ // Finds iterator of object with provided objId inside
+
+ TAdaptiveLock FilesLock_; // Lock to serialize access to internal data structures associated with files content
+ TAdaptiveLock ToSaveLock_; // Lock to serialize access to save queue
+ TAdaptiveLock ToLoadLock_; // Lock to serialize access to load queue
+ TAdaptiveLock ToDeleteLock_; // Lock to serialize access to delete queue
+ TString Name_; // Name of the namespace
+ ui32 Id_ = 0; // Internal id of the namespace
+ std::deque<TSaveTask> ToSave_; // Queue to spill data
+ std::deque<TLoadTask> ToLoad_; // Queue to load spilled data
+ std::deque<TBaseTask> ToDelete_; // Queue to delete spilled data
+ std::unordered_map<TString, ui32> LastObjs_; // Latest version of objects with particular name
+ std::unordered_map<TString, std::vector<ui32>> LastStreamIds_; // Latest buff id of particular stream
+ std::unordered_map<ui32, TSessionDesc> Sessions_; // Sessions descriptions by session id key
+ std::unordered_map<ui32, std::atomic<ui64>> SessionDataProvided_; // Data provided to spill per session
+ std::unordered_map<ui32, std::atomic<ui64>> SessionDataSpilled_; // Data spilled to storage from memory per session
+ std::unordered_map<ui32, std::atomic<ui64>> SessionDataLoadedFromStorage_; // Data loaded from storage to memory per session
+ std::unordered_map<ui32, std::atomic<ui64>> SessionDataLoadedFromMemory_; // Data loaded from memory to memory per session
+ TAtomicSharedPtr<ISpillStorage> StorageI_; // Interface to storage to work with namespace
+ TAtomicSharedPtr<ISpillFile> CurrSpillMetaFile_ = nullptr; // Current spill meta file to write data
+ TAtomicSharedPtr<ISpillFile> CurrSpillDataFile_ = nullptr; // Current spill file to write data
+ ui32 CurrSpillFileId_ = 0; // Current id of spill file in the namespace
+ std::atomic<ui32> NextObjId_ = 0; // Next object id in the namespace
+ std::map<ui32, TAtomicSharedPtr<TMetaFileAttributes> > SpillMetaFiles_; // Current open spill meta files for particular namespace
+ std::set<ui32> SpillFilesIdToDelete_; // Id of spill files which need to be deleted
+
+};
+
+
+}
+}
diff --git a/ydb/library/yql/core/spilling/namespaces_list.cpp b/ydb/library/yql/core/spilling/namespaces_list.cpp
new file mode 100644
index 00000000000..977fd9dbefe
--- /dev/null
+++ b/ydb/library/yql/core/spilling/namespaces_list.cpp
@@ -0,0 +1,167 @@
+#include "namespaces_list.h"
+#include "interface/spilling.h"
+#include "storage/storage.h"
+
+
+#include <map>
+
+
+namespace NYql {
+namespace NSpilling {
+
+
+TAtomicSharedPtr<TNamespaceCache> TNamespacesList::GetNamespaceCache(const TString& name){
+ TAtomicSharedPtr<TNamespaceCache> res;
+ with_lock(NamespacesMapLock_) {
+ if ( auto found = NamespacesMap_.find(name); found != NamespacesMap_.end() ) {
+ res = found->second;
+ } else {
+ CurrNamespaceId_++;
+ res = MakeAtomicShared<TNamespaceCache>(name, CurrNamespaceId_, StorageI_);
+ NamespacesMap_.emplace(name, res);
+ }
+ }
+ return res;
+}
+
+
+void TNamespacesList::PushNamespaceToProcess(const TString& name, ui32 sessionId) {
+ with_lock(QueueLock_) {
+ NamespacesQueue_.push_back(name);
+ if (ActiveWorkerThreads_ < NamespacesQueue_.size()) {
+ WorkingSemaphore_.release();
+ }
+ }
+ with_lock(SessionsLock_) {
+ SessionNamespaces_[sessionId].emplace(name);
+ }
+}
+
+bool TNamespacesList::PopNamespaceToProcess(TString& name) {
+ bool res = false;
+ with_lock(QueueLock_) {
+ if ( NamespacesQueue_.size() > 0) {
+ name = NamespacesQueue_.front();
+ NamespacesQueue_.pop_front();
+ res = true;
+ } else {
+ res = false;
+ }
+ }
+ return res;
+}
+
+bool TNamespacesList::WaitForTask() {
+ std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
+ auto diff = std::chrono::milliseconds(1000);
+ ActiveWorkerThreads_--;
+ bool task = WorkingSemaphore_.try_acquire_for(diff);
+ ActiveWorkerThreads_++;
+ return task;
+}
+
+void TNamespacesList::PerformGarbageCollection() {
+
+ with_lock(GarbageCollectionLock_) {
+ if ( NamespacesForGarbageCollection_.size() == 0 ) {
+ NamespacesForGarbageCollection_ = StorageI_->GetNamespaces();
+ }
+ }
+
+ bool needGarbageCollection = true;
+
+ if (GarbageCollectionThreads_ <= MaxGarbageCollectionThreads ) {
+ GarbageCollectionThreads_++;
+ } else {
+ return;
+ }
+
+ while (needGarbageCollection) {
+ TString ns;
+ with_lock( GarbageCollectionLock_ ) {
+ if ( NamespacesForGarbageCollection_.size() > 0 ) {
+ ns = NamespacesForGarbageCollection_.back();
+ NamespacesForGarbageCollection_.pop_back();
+ } else {
+ needGarbageCollection = false;
+ break;
+ }
+ }
+
+ TAtomicSharedPtr<TNamespaceCache> nsc;
+ nsc = GetNamespaceCache(ns);
+ nsc->GarbageCollection();
+
+ }
+
+ GarbageCollectionThreads_--;
+
+
+}
+
+void TNamespacesList::RemoveNamespaceCache(const TString& name){
+ with_lock(NamespacesMapLock_) {
+ NamespacesMap_.erase(name);
+ }
+}
+
+bool TNamespacesList::CheckForStop() {
+ bool res = false;
+ with_lock(NamespacesMapLock_) {
+ res = StopFlag_;
+ }
+ return res;
+}
+
+void TNamespacesList::Stop() {
+ with_lock(NamespacesMapLock_) {
+ StopFlag_ = true;
+ for (ui32 i = 0; i < MaxThreadPoolSize; i++ ) {
+ WorkingSemaphore_.release();
+ }
+ }
+}
+
+TSessionDataStat TNamespacesList::GetSessionDataStat(ui32 sessionId) {
+ TSessionDataStat res;
+ with_lock(SessionsLock_) {
+ auto found = SessionNamespaces_.find(sessionId);
+ if (found != SessionNamespaces_.end()) {
+ for (auto& it: found->second) {
+ TSessionDataStat ns_res;
+ auto nsc = GetNamespaceCache(it);
+ ns_res = nsc->GetSessionStat(sessionId);
+ res.Provided += ns_res.Provided;
+ res.InMemory += ns_res.InMemory;
+ res.Spilled += ns_res.Spilled;
+ res.LoadedFromStorage += ns_res.LoadedFromStorage;
+ res.LoadedFromMemory += ns_res.LoadedFromMemory;
+ }
+ }
+ }
+ return res;
+}
+
+void TNamespacesList::CloseSession(ui32 sessionId) {
+ with_lock(SessionsLock_) {
+ auto found = SessionNamespaces_.find(sessionId);
+ if (found != SessionNamespaces_.end()) {
+ for (auto& it: found->second) {
+ TSessionDataStat ns_res;
+ auto nsc = GetNamespaceCache(it);
+ nsc->CloseSession(sessionId);
+ }
+ }
+ }
+
+
+}
+
+TNamespacesList::TNamespacesList(ui32 threadPoolSize, TAtomicSharedPtr<ISpillStorage> storageI) :
+ ActiveWorkerThreads_(threadPoolSize),
+ StorageI_(storageI) {
+ LastGarbageCollection_ = std::chrono::steady_clock::now();
+}
+
+}
+}
diff --git a/ydb/library/yql/core/spilling/namespaces_list.h b/ydb/library/yql/core/spilling/namespaces_list.h
new file mode 100644
index 00000000000..e1b3fb6f0fd
--- /dev/null
+++ b/ydb/library/yql/core/spilling/namespaces_list.h
@@ -0,0 +1,54 @@
+#include "interface/spilling.h"
+#include "storage/storage.h"
+#include "namespace_cache.h"
+
+#include <map>
+#include <deque>
+#include <semaphore>
+#include <set>
+
+
+namespace NYql {
+namespace NSpilling {
+
+const ui32 DefaultThreadPoolSize = 10;
+const ui32 MaxThreadPoolSize = 30;
+
+const ui32 GarbageCollectionPeriodMs = 30000; // Default timeout for garbage collection activity (milliseconds)
+const ui32 MaxGarbageCollectionThreads = 3; // Maximum number of threads to perform garbage collection
+
+class TNamespacesList {
+public:
+ TAtomicSharedPtr<TNamespaceCache> GetNamespaceCache(const TString& name);
+ void PushNamespaceToProcess(const TString& name, ui32 sessionId); // Adds particular namespace for async processing
+ bool PopNamespaceToProcess(TString& name); // Returns true if there is next namespace for processing. Namespace is returned in TString& name argument.
+ bool WaitForTask(); // Waits for the next task for worker thread
+ void PerformGarbageCollection(); // Performs garbage collection with GarbageCollectionPeriodMs
+ void RemoveNamespaceCache(const TString& name);
+ bool CheckForStop(); // Checks if worker thread need to be stopped
+ void Stop(); // Signal to stop workers threads
+ TSessionDataStat GetSessionDataStat(ui32 SessionId); // Returns statistics for data spilled and loaded during session
+ void CloseSession(ui32 SessionId); // Closes session and clears all resources
+ TNamespacesList(ui32 threadPoolSize, TAtomicSharedPtr<ISpillStorage> storageI);
+private:
+ TAdaptiveLock NamespacesMapLock_; // Lock to serialize access to internal map data structures
+ TAdaptiveLock QueueLock_; // Lock for NamespacesQueue
+ TAdaptiveLock SessionsLock_; // Lock for SessionNamespaces lock
+ TAdaptiveLock GarbageCollectionLock_; // Lock for Garbage collection activity
+ std::atomic<ui32> ActiveWorkerThreads_ = 0; // Number of current active working threads
+ std::atomic<ui32> GarbageCollectionThreads_ = 0; // Number of current active working threads
+ std::counting_semaphore<> WorkingSemaphore_{MaxThreadPoolSize}; // Semaphore to wait for the worker threads activation
+ ui32 CurrNamespaceId_ = 1; // Current namespace id to assign to new namespace
+ std::map<TString, TAtomicSharedPtr<TNamespaceCache>> NamespacesMap_; // Map to find required namespace cache by name
+ std::map<ui32, std::set<TString> > SessionNamespaces_; // Set of all namespace names for particular session
+ std::deque<TString> NamespacesQueue_; // Queue to process namespaces with pending tasks by thread pool
+ bool StopFlag_ = false;
+ TAtomicSharedPtr<ISpillStorage> StorageI_; // Interface to storage to work with namespace
+ std::chrono::steady_clock::time_point LastGarbageCollection_; // Last time when namespace garbage collection was performed
+ std::vector<TString> NamespacesForGarbageCollection_; // List of namespaces to perform garbage collection
+
+};
+
+
+}
+}
diff --git a/ydb/library/yql/core/spilling/spilling_imp.cpp b/ydb/library/yql/core/spilling/spilling_imp.cpp
new file mode 100644
index 00000000000..f6b5f74c9ff
--- /dev/null
+++ b/ydb/library/yql/core/spilling/spilling_imp.cpp
@@ -0,0 +1,245 @@
+#include "spilling_imp.h"
+#include "interface/spilling.h"
+#include "storage/storage.h"
+
+#include <ydb/library/yql/utils/log/log.h>
+
+#include <random>
+#include <util/folder/path.h>
+#include <filesystem>
+#include <iostream>
+#include <limits>
+#include <unordered_map>
+#include <chrono>
+
+#include <util/string/split.h>
+
+#include <util/folder/dirut.h>
+#include <util/generic/algorithm.h>
+#include <util/generic/vector.h>
+#include <util/generic/yexception.h>
+#include <util/generic/ptr.h>
+#include <util/generic/utility.h>
+#include <util/system/file.h>
+#include <util/system/file_lock.h>
+#include <util/system/fs.h>
+#include <util/system/maxlen.h>
+#include <util/system/mutex.h>
+#include <util/system/utime.h>
+#include <util/system/thread.h>
+
+
+
+namespace NYql {
+namespace NSpilling {
+
+THolder<ISession> TTempStorageProxyImp::CreateSession() {
+ if (CurrSessId_ == std::numeric_limits<ui32>::max() ) {
+ CurrSessId_ = 1;
+ }
+ ui32 sessId = CurrSessId_.fetch_add(1);
+ return MakeHolder<TSessionImp>(sessId, NsList_, StorageI_);
+
+}
+
+THolder<IObjectsIterator> TTempStorageProxyImp::CreateIterator(
+ const TMaybe<TString>& objNamespace,
+ const TMaybe<TString>& objName,
+ bool onlyValid) {
+
+ return MakeHolder<TStorageIteratorImp>(objNamespace, objName, onlyValid);
+}
+
+TTempStorageExecutionPolicy TTempStorageProxyImp::ExecutionPolicy() {
+ return Policy_;
+}
+
+TOperationResults TTempStorageProxyImp::SetExecutionPolicy(const TTempStorageExecutionPolicy & policy) {
+ Policy_ = policy;
+}
+
+NThreading::TFuture<TOperationResults> TTempStorageProxyImp::Delete(const TString & objNamespace, const TMaybe<TString> & name) {
+ NThreading::TFuture<TOperationResults> res;
+ return res;
+}
+
+
+TOperationResults TTempStorageProxyImp::LastOperationResults(){
+ return OperationResults_;
+}
+
+ui32 Ui32Rand(const ui32 & min, const ui32 & max) {
+ static thread_local std::mt19937 generator;
+ std::uniform_int_distribution<ui32> distribution(min,max);
+ return distribution(generator);
+}
+
+// Worker function in threads pool
+void ProcessThreadPoolTasks(TAtomicSharedPtr<TNamespacesList> nsl, TAtomicSharedPtr<ISpillStorage> sti){
+ bool stopFlag = false;
+ bool haveNamespaceToProcess = false;
+ TString namespaceToProcess;
+ ui32 periodShift = Ui32Rand(0, GarbageCollectionPeriodMs / 8);
+
+ std::chrono::steady_clock::time_point lastGarbageCollection = std::chrono::steady_clock::now();
+
+ TThread::TId this_id = TThread::CurrentThreadNumericId();
+
+ while( !stopFlag ) {
+ try {
+ bool task = nsl->WaitForTask();
+
+ std::chrono::steady_clock::time_point now = std::chrono::steady_clock::now();
+ ui64 period = std::chrono::duration_cast<std::chrono::milliseconds>(now - lastGarbageCollection).count();
+ if ( period > GarbageCollectionPeriodMs + periodShift ) {
+
+ if ( this_id % DefaultThreadPoolSize <= MaxGarbageCollectionThreads) {
+ lastGarbageCollection = now;
+ nsl->PerformGarbageCollection();
+ now = std::chrono::steady_clock::now();
+ ui64 execTime = std::chrono::duration_cast<std::chrono::microseconds>(now - lastGarbageCollection).count();
+ YQL_LOG(INFO) << "Thread " << this_id << " garbage collection (microseconds): " << execTime << Endl;
+ }
+ }
+
+ haveNamespaceToProcess = nsl->PopNamespaceToProcess(namespaceToProcess);
+ if (haveNamespaceToProcess) {
+ TAtomicSharedPtr<TNamespaceCache> nsc = nsl->GetNamespaceCache(namespaceToProcess);
+ nsc->Dispatch();
+ } else {
+ }
+ stopFlag = nsl->CheckForStop();
+ } catch(...) {
+
+ YQL_LOG(ERROR) << "Error happened..." << FormatCurrentException() << Endl;
+ }
+ }
+ YQL_LOG(INFO) << "Stopping spilling thread " << this_id << " ..." << Endl;
+}
+
+
+TTempStorageProxyImp::TTempStorageProxyImp(const TFileStorageConfig & config, const TTempStorageExecutionPolicy & policy, THolder<ISpillStorage>&& storage) :
+ StorageConfig_(config),
+ Policy_(policy),
+ StorageI_(std::move(storage))
+{
+ OperationResults_.Status = EOperationStatus::Success;
+ NsList_ = MakeAtomicShared<TNamespacesList>(DefaultThreadPoolSize, StorageI_);
+ for (ui32 i = 0; i < DefaultThreadPoolSize; i++) {
+ ThreadPool_.push_back(std::thread(ProcessThreadPoolTasks, NsList_, StorageI_));
+ }
+}
+
+
+TTempStorageProxyImp::~TTempStorageProxyImp(){
+ YQL_LOG(INFO) << "Stopping nslist ..." << Endl;
+ NsList_->Stop();
+ YQL_LOG(INFO) << "Stopping threads ..." << Endl;
+ for(auto & t: ThreadPool_) {
+ t.join();
+ }
+}
+
+
+
+NThreading::TFuture<TOperationResults> TSessionImp::Save(const TString & objNamespace, const TString & name, TBuffer && buf){
+ TAtomicSharedPtr<TNamespaceCache> nsc = NsList_->GetNamespaceCache(objNamespace);
+ NsList_->PushNamespaceToProcess(objNamespace, SessionId_);
+ return nsc->Save(name, std::move(buf), SessionId_);
+}
+
+NThreading::TFuture<TLoadOperationResults> TSessionImp::Load(const TString & objNamespace, const TString & name, EObjectsLifetime objLifetime ){
+ TAtomicSharedPtr<TNamespaceCache> nsc = NsList_->GetNamespaceCache(objNamespace);
+ NsList_->PushNamespaceToProcess(objNamespace, SessionId_);
+ return nsc->Load(name, SessionId_, objLifetime);
+}
+
+
+TSessionDataStat TSessionImp::GetSessionDataStat(){
+ return NsList_->GetSessionDataStat(SessionId_);
+}
+
+TSessionExecutionPolicy TSessionImp::ExecutionPolicy(){
+ return Policy_;
+}
+
+
+TOperationResults TSessionImp::SetExecutionPolicy(const TSessionExecutionPolicy& policy){
+ TOperationResults res;
+ Policy_ = policy;
+ return res;
+}
+
+std::pair<THolder<IStream>, TOperationResults> TSessionImp::OpenStream(const TString& objNamespace, const TString& streamName ) {
+ THolder<TStreamImp> sti = MakeHolder<TStreamImp>(objNamespace, streamName, NsList_, SessionId_);
+ TOperationResults res;
+ return std::make_pair<THolder<IStream>, TOperationResults> (std::move(sti), std::move(res));
+}
+
+
+TSessionImp::TSessionImp(ui32 sessionId, TAtomicSharedPtr<TNamespacesList> nsList, TAtomicSharedPtr<ISpillStorage> storage) :
+ SessionId_(sessionId),
+ NsList_(nsList),
+ StorageI_(storage) {
+
+}
+
+TStreamImp::TStreamImp(const TString& objNamespace, const TString& streamName, TAtomicSharedPtr<TNamespacesList> nsList, ui32 sessionId ) :
+ Namespace_(objNamespace),
+ Name_(streamName),
+ NsList_(nsList),
+ SessionId_(sessionId) {};
+
+TSessionImp::~TSessionImp(){
+ NsList_->CloseSession(SessionId_);
+}
+
+
+
+bool TStorageIteratorImp::Next(TTempObjectDesc & object) {
+ return false;
+}
+
+TStorageIteratorImp::TStorageIteratorImp(const TMaybe<TString>& objNamespace, const TMaybe<TString>& objName, bool onlyValid) {
+}
+
+
+NThreading::TFuture<TOperationResults> TStreamImp::Save(TBuffer&& buf){
+ TAtomicSharedPtr<TNamespaceCache> nsc = NsList_->GetNamespaceCache(Namespace_);
+ NsList_->PushNamespaceToProcess(Namespace_, SessionId_);
+ return nsc->Save(Name_, std::move(buf), SessionId_, true);
+}
+
+ui64 TStreamImp::Size() {
+ TAtomicSharedPtr<TNamespaceCache> nsc = NsList_->GetNamespaceCache(Namespace_);
+ return nsc->StreamSize(Name_);
+
+}
+
+NThreading::TFuture<TLoadOperationResults> TStreamImp::Load(ui64 bufferId, EObjectsLifetime objLifetime ) {
+ TAtomicSharedPtr<TNamespaceCache> nsc = NsList_->GetNamespaceCache(Namespace_);
+ NsList_->PushNamespaceToProcess(Namespace_, SessionId_);
+ return nsc->Load(Name_, SessionId_, objLifetime, true, bufferId);
+
+
+}
+
+NThreading::TFuture<TOperationResults> TStreamImp::Close() {
+ NThreading::TFuture<TOperationResults> res;
+ return res;
+}
+
+
+
+
+std::pair< THolder<ITempStorageProxy>, TOperationResults > CreateFileStorageProxy(const TFileStorageConfig & config, const TTempStorageExecutionPolicy & policy ) {
+
+ std::pair< THolder<ISpillStorage>, TOperationResults > sps = OpenFileStorageForSpilling(config);
+ THolder<TTempStorageProxyImp> sp = MakeHolder<TTempStorageProxyImp>(config, policy, std::move(sps.first));
+ TOperationResults res = sps.second;
+ return std::make_pair< THolder<ITempStorageProxy>, TOperationResults >( std::move(sp), std::move(res) );
+}
+
+
+}
+} \ No newline at end of file
diff --git a/ydb/library/yql/core/spilling/spilling_imp.h b/ydb/library/yql/core/spilling/spilling_imp.h
new file mode 100644
index 00000000000..271163c1564
--- /dev/null
+++ b/ydb/library/yql/core/spilling/spilling_imp.h
@@ -0,0 +1,98 @@
+#include "interface/spilling.h"
+#include "storage/storage.h"
+#include "namespaces_list.h"
+
+#include <thread>
+#include <set>
+
+namespace NYql {
+namespace NSpilling {
+
+const ui64 MagicForFileRecord = 0xA957248FEED9E4CE;
+
+
+
+
+
+class TTempStorageProxyImp : public ITempStorageProxy {
+public:
+ THolder<ISession> CreateSession();
+ THolder<IObjectsIterator> CreateIterator( const TMaybe<TString>& objNamespace = TMaybe<TString>(),
+ const TMaybe<TString>& objName = TMaybe<TString>(),
+ bool onlyValid = true);
+ TTempStorageExecutionPolicy ExecutionPolicy();
+ TOperationResults SetExecutionPolicy(const TTempStorageExecutionPolicy& policy);
+ NThreading::TFuture<TOperationResults> Delete(const TString& objNamespace, const TMaybe<TString>& name);
+ TOperationResults LastOperationResults();
+ TTempStorageProxyImp(const TFileStorageConfig & config, const TTempStorageExecutionPolicy & policy, THolder<ISpillStorage>&& storage);
+ ~TTempStorageProxyImp();
+
+private:
+ TFileStorageConfig StorageConfig_;
+ TTempStorageExecutionPolicy Policy_;
+ TAtomicSharedPtr<ISpillStorage> StorageI_;
+ TOperationResults OperationResults_; // Last operation results
+ std::atomic<ui32> CurrSessId_ = 1; // Current session id to assign new session
+ std::vector<std::thread> ThreadPool_; // Thread pool to process spilling tasks
+ TAtomicSharedPtr<TNamespacesList> NsList_; // List of namespaces to forward requests
+
+};
+
+// Worker function in threads pool
+void ProcessThreadPoolTasks(TAtomicSharedPtr<TNamespacesList> nsl, TAtomicSharedPtr<ISpillStorage> sti);
+
+
+// Class to store session object ids
+class TSessionIds {
+ ui32 NsId_ = 0;
+ ui32 ObjId_ = 0;
+};
+
+class TSessionImp: public ISession {
+public:
+ NThreading::TFuture<TOperationResults> Save(const TString & objNamespace, const TString & name, TBuffer && buf);
+ NThreading::TFuture<TLoadOperationResults> Load(const TString & objNamespace, const TString & name, EObjectsLifetime objLifetime = EObjectsLifetime::DeleteAfterLoad );
+ TSessionDataStat GetSessionDataStat();
+ TSessionExecutionPolicy ExecutionPolicy();
+ TOperationResults SetExecutionPolicy(const TSessionExecutionPolicy& policy);
+ std::pair<THolder<IStream>, TOperationResults> OpenStream(const TString& objNamespace, const TString& streamName );
+ TSessionImp(ui32 sessionId, TAtomicSharedPtr<TNamespacesList> nsList, TAtomicSharedPtr<ISpillStorage> storage);
+ ~TSessionImp();
+
+private:
+ ui32 SessionId_;
+ TSessionExecutionPolicy Policy_;
+ TAtomicSharedPtr<TNamespacesList> NsList_; // List of namespaces to forward requests
+ TAtomicSharedPtr<ISpillStorage> StorageI_; // Storage interface to spill session data
+};
+
+class TStorageIteratorImp: public IObjectsIterator {
+public:
+ bool Next(TTempObjectDesc & object);
+ TStorageIteratorImp(const TMaybe<TString>& objNamespace = TMaybe<TString>(),
+ const TMaybe<TString>& objName = TMaybe<TString>(),
+ bool onlyValid = true);
+private:
+ bool NameSpacesIterator_ = false;
+ TVector<TString> ObjNamespaces_;
+ TVector<TString>::const_iterator CurrNamespace_;
+};
+
+
+class TStreamImp: public IStream {
+public:
+ NThreading::TFuture<TOperationResults> Save(TBuffer&& buf);
+ ui64 Size();
+ NThreading::TFuture<TLoadOperationResults> Load(ui64 bufferId = 0, EObjectsLifetime objLifetime = EObjectsLifetime::DeleteAfterLoad);
+ NThreading::TFuture<TOperationResults> Close();
+ TStreamImp(const TString& objNamespace, const TString& streamName, TAtomicSharedPtr<TNamespacesList> nsList, ui32 sessionId );
+private:
+ TString Namespace_;
+ TString Name_;
+ TAtomicSharedPtr<TNamespacesList> NsList_;
+ ui32 SessionId_;
+};
+
+
+}
+} \ No newline at end of file
diff --git a/ydb/library/yql/core/spilling/storage/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/spilling/storage/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..cd28502e3b3
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,31 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(file_storage)
+add_subdirectory(ut)
+
+add_library(core-spilling-storage)
+target_compile_options(core-spilling-storage PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(core-spilling-storage PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ spilling-storage-file_storage
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(core-spilling-storage PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/storage/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/spilling/storage/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..4150b7499b7
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,32 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(file_storage)
+add_subdirectory(ut)
+
+add_library(core-spilling-storage)
+target_compile_options(core-spilling-storage PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(core-spilling-storage PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ spilling-storage-file_storage
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(core-spilling-storage PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/storage/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/spilling/storage/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..4150b7499b7
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,32 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(file_storage)
+add_subdirectory(ut)
+
+add_library(core-spilling-storage)
+target_compile_options(core-spilling-storage PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(core-spilling-storage PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ spilling-storage-file_storage
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(core-spilling-storage PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/storage/CMakeLists.txt b/ydb/library/yql/core/spilling/storage/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/spilling/storage/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/spilling/storage/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..cd28502e3b3
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,31 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+add_subdirectory(file_storage)
+add_subdirectory(ut)
+
+add_library(core-spilling-storage)
+target_compile_options(core-spilling-storage PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(core-spilling-storage PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ spilling-storage-file_storage
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(core-spilling-storage PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..9028583a24d
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,29 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(spilling-storage-file_storage)
+target_compile_options(spilling-storage-file_storage PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(spilling-storage-file_storage PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yql-utils-log
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(spilling-storage-file_storage PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..e86132790a7
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,30 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(spilling-storage-file_storage)
+target_compile_options(spilling-storage-file_storage PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(spilling-storage-file_storage PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yql-utils-log
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(spilling-storage-file_storage PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..e86132790a7
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,30 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(spilling-storage-file_storage)
+target_compile_options(spilling-storage-file_storage PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(spilling-storage-file_storage PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ yql-utils-log
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(spilling-storage-file_storage PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.txt b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..9028583a24d
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/file_storage/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,29 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(spilling-storage-file_storage)
+target_compile_options(spilling-storage-file_storage PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_link_libraries(spilling-storage-file_storage PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ yql-utils-log
+ yql-minikql-codegen
+ llvm12-lib-IR
+ lib-ExecutionEngine-MCJIT
+ llvm12-lib-Linker
+ lib-Target-X86
+ Target-X86-AsmParser
+ lib-Transforms-IPO
+)
+target_sources(spilling-storage-file_storage PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp
+)
diff --git a/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp b/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp
new file mode 100644
index 00000000000..29c8ab020ea
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/file_storage/file_storage.cpp
@@ -0,0 +1,260 @@
+#pragma once
+
+#include <library/cpp/threading/future/async.h>
+#include <util/folder/path.h>
+#include <filesystem>
+#include <iostream>
+#include <limits>
+#include <unordered_map>
+#include <chrono>
+
+#include <util/string/split.h>
+
+#include <util/folder/dirut.h>
+#include <util/generic/algorithm.h>
+#include <util/generic/vector.h>
+#include <util/generic/yexception.h>
+#include <util/generic/ptr.h>
+#include <util/generic/utility.h>
+#include <util/system/file.h>
+#include <util/system/file_lock.h>
+#include <util/system/fs.h>
+#include <util/system/maxlen.h>
+#include <util/system/mutex.h>
+#include <util/system/utime.h>
+#include <util/system/thread.h>
+
+#include <ydb/library/yql/core/spilling/interface/spilling.h>
+#include <ydb/library/yql/core/spilling/storage/storage.h>
+
+
+#include <ydb/library/yql/utils/log/log.h>
+
+
+namespace NYql {
+namespace NSpilling {
+
+// Class to implement ISpillStorage interface based on file system storage
+class FileSpillStorage: public ISpillStorage {
+public:
+
+ ui64 GetCurrSize() {return 0;} // Returns current size of spill storage
+ // Returns full list of namespaces for current spill storage. Number of namespaces should be reasonable (no more than 1000)
+ TVector<TString> GetNamespaces();
+
+ // Returns list of file names for namespace
+ TVector<TString> GetNamespaceFiles(const TString& ns);
+
+ THolder<ISpillFile> CreateSpillFile(const TString& ns, const TString& fn, ui32 reserveStep);
+
+ TOperationResults LastOperationResults();
+ FileSpillStorage (const TFileStorageConfig& config);
+
+private:
+ TFsPath RootPath_; // Root path to storage directory
+ TOperationResults OperationResults_; // Last operation results
+
+ bool RootPathExists(); // Returns true if root path exists for spill storage
+
+};
+
+
+// Class to implement ISpillFile interface based on file system files.
+class FsSpillFile: public ISpillFile {
+public:
+ TString GetName() {return Name_;};
+ bool IsLocked();
+ ui64 Reserve(ui32 size);
+ void Write(ui32 offset, const char * data, ui32 bytes);
+ void Seek(ui32 offset);
+ i32 Read(ui32 offset, char* buf, ui32 len);
+ void Delete();
+ FsSpillFile(const TString& name, EOpenMode oMode, ui32 reserveStep);
+private:
+ TString Name_;
+ TFile File_;
+ TFileLock FileLock_;
+ bool Locked_ = false;
+ std::atomic<ui64> TotalSpace_ = 0;
+ std::atomic<ui64> ReservedSpace_ = 0;
+ ui32 ReserveStep_ = 10000000; // File is incremented by 10 MB chunks
+};
+
+
+TVector<TString> FileSpillStorage::GetNamespaces() {
+ TVector<TString> res;
+ std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
+
+ if (!RootPathExists()) {
+ OperationResults_.Status = EOperationStatus::CannotOpenStorageProxy;
+ std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
+ OperationResults_.ErrorString = "Root directory for temp storage path: " + TString(RootPath_) + " does not exist";
+ return res;
+ }
+
+
+ TVector<TFsPath> dirEntries;
+ RootPath_.List(dirEntries);
+ for (const auto & p: dirEntries) {
+ if (p.IsDirectory() ) {
+ res.push_back(p.GetName());
+ }
+ }
+
+ OperationResults_.Status = EOperationStatus::Success;
+ OperationResults_.ErrorString.clear();
+
+ return res;
+
+}
+
+TVector<TString> FileSpillStorage::GetNamespaceFiles(const TString& ns) {
+ TVector<TString> res;
+
+ if (!RootPathExists()) {
+ OperationResults_.Status = EOperationStatus::CannotOpenStorageProxy;
+ OperationResults_.ErrorString = "Root directory for temp storage path: " + TString(RootPath_) + " does not exist";
+ return res;
+ }
+
+ TFsPath child = RootPath_.Child(ns);
+ if ( !child.Exists()) {
+ return res;
+ }
+ TVector<TFsPath> dirEntries;
+ child.List(dirEntries);
+ for (const auto & p: dirEntries) {
+ TVector<TString> splitted;
+ size_t tokenCount = Split(p.GetName(), ".", splitted);
+ if (tokenCount >= 3 && tokenCount <= 4 && splitted[0] == "ydbspl" ) {
+ ui64 ind = 0;
+ if (TryFromString(splitted[1], ind)) {
+ }
+ res.push_back(p.GetName());
+ }
+
+ }
+
+ OperationResults_.Status = EOperationStatus::Success;
+ OperationResults_.ErrorString.clear();
+
+ return res;
+
+}
+
+THolder<ISpillFile> FileSpillStorage::CreateSpillFile(const TString& ns, const TString& fn, ui32 reserveStep) {
+ TFsPath nsdir = RootPath_.Child(ns);
+ if (!nsdir.Exists()) {
+ nsdir.MkDir();
+ }
+ TFsPath filePath = nsdir.Child(fn);
+ return MakeHolder<FsSpillFile>(filePath.GetPath(),
+ EOpenModeFlag::OpenAlways | EOpenModeFlag::RdWr , reserveStep);
+}
+
+
+bool FileSpillStorage::RootPathExists() {
+ if (RootPath_.IsDirectory()) {
+ return true;
+ } else {
+ return false;
+ }
+}
+
+TOperationResults FileSpillStorage::LastOperationResults() {
+ return OperationResults_;
+}
+
+FileSpillStorage::FileSpillStorage (const TFileStorageConfig& config) {
+
+ std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
+
+ RootPath_ = TFsPath(config.Path);
+
+ if (!RootPathExists()) {
+ OperationResults_.Status = EOperationStatus::CannotOpenStorageProxy;
+ OperationResults_.ErrorString = "Temp storage path: " + config.Path + " does not exist";
+ return;
+ }
+
+
+ TVector<TFsPath> dirEntries;
+ RootPath_.List(dirEntries);
+ for (const auto & p: dirEntries) {
+ TVector<TString> splitted;
+ size_t tokenCount = Split(p.GetName(), ".", splitted);
+ if (tokenCount == 3 && splitted[0] == "ydbspl" ) {
+ ui64 ind = 0;
+ if (TryFromString(splitted[1], ind)) {
+ }
+ }
+ }
+
+ OperationResults_.Status = EOperationStatus::Success;
+ std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
+ OperationResults_.ErrorString.clear();
+
+
+
+}
+
+
+std::pair< THolder<ISpillStorage>, TOperationResults > OpenFileStorageForSpilling(const TFileStorageConfig & config ) {
+ THolder<FileSpillStorage> sp = MakeHolder<FileSpillStorage>(config);
+ TOperationResults res = sp->LastOperationResults();
+ return std::make_pair< THolder<ISpillStorage>, TOperationResults >( std::move(sp), std::move(res) );
+}
+
+
+bool FsSpillFile::IsLocked() {
+ return Locked_;
+}
+
+ui64 FsSpillFile::Reserve(ui32 size) {
+ ui64 offset = ReservedSpace_.fetch_add(size);
+ while ( (offset + size) >= TotalSpace_ ) {
+ File_.Resize(offset + size + ReserveStep_);
+ TotalSpace_ = File_.GetLength();
+ }
+
+ return offset;
+}
+
+void FsSpillFile::Write(ui32 offset, const char * data, ui32 bytes) {
+ File_.Pwrite(data, bytes, offset);
+}
+
+void FsSpillFile::Seek(ui32 offset) {
+ File_.Seek(offset, SeekDir::sSet);
+}
+
+i32 FsSpillFile::Read(ui32 offset, char* buf, ui32 len) {
+ return File_.Pread(buf, len, offset);
+}
+
+void FsSpillFile::Delete() {
+ TFsPath path(Name_);
+ YQL_LOG(INFO) << "Deleting: " << Name_ << Endl;
+ path.ForceDelete();
+}
+
+FsSpillFile::FsSpillFile(const TString& name, EOpenMode oMode, ui32 reserveStep) :
+ Name_(name),
+ File_(name, oMode),
+ FileLock_(name),
+ ReserveStep_(reserveStep)
+{
+ Locked_ = FileLock_.TryAcquire();
+ if (Locked_) {
+ ui64 fsize = File_.GetLength();
+ if (fsize >= std::numeric_limits<ui32>::max()) {
+
+ }
+ TotalSpace_ = (ui32) fsize;
+ ReservedSpace_ = fsize;
+ }
+}
+
+
+}
+} \ No newline at end of file
diff --git a/ydb/library/yql/core/spilling/storage/file_storage/ya.make b/ydb/library/yql/core/spilling/storage/file_storage/ya.make
new file mode 100644
index 00000000000..bb62430431c
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/file_storage/ya.make
@@ -0,0 +1,32 @@
+LIBRARY()
+
+SRCS(
+ file_storage.cpp
+)
+
+PEERDIR(
+ ydb/library/yql/utils/log
+)
+
+NO_COMPILER_WARNINGS()
+
+IF (NOT MKQL_DISABLE_CODEGEN)
+ PEERDIR(
+ ydb/library/yql/minikql/codegen
+ contrib/libs/llvm12/lib/IR
+ contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
+ contrib/libs/llvm12/lib/Linker
+ contrib/libs/llvm12/lib/Target/X86
+ contrib/libs/llvm12/lib/Target/X86/AsmParser
+ contrib/libs/llvm12/lib/Transforms/IPO
+ )
+ELSE()
+ CFLAGS(
+ -DMKQL_DISABLE_CODEGEN
+ )
+ENDIF()
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
diff --git a/ydb/library/yql/core/spilling/storage/storage.cpp b/ydb/library/yql/core/spilling/storage/storage.cpp
new file mode 100644
index 00000000000..874e7ac91f5
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/storage.cpp
@@ -0,0 +1,141 @@
+#include "storage.h"
+
+#include <format>
+#include <contrib/libs/xxhash/xxhash.h>
+#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/library/yql/utils/log/log.h>
+
+namespace NYql {
+namespace NSpilling {
+
+inline ui32 CalcUi32StrLen(ui32 bytes) {
+ if (!bytes) return 0;
+ return bytes / sizeof(ui32) + 1;
+}
+
+EOperationType TSpillMetaRecord::GetOpType() {
+ ui32 opType = (Meta1_ & Meta1OperationBitMask) >> 28; // Calculating record type
+ return (EOperationType) opType;
+
+}
+
+
+ui32 TSpillMetaRecord::GetNameSize() {
+ return (Meta1_ & (Meta1OperationStrLen) );
+}
+
+
+ui32 TSpillMetaRecord::Size() {
+ ui32 res;
+
+ res = 7 * sizeof(ui32); // Meta size + total record size
+ res += (CalcUi32StrLen(Name_.size() ) ) * sizeof(ui32);
+
+ EOperationType eOpType = GetOpType();
+
+ switch (eOpType) {
+ case EOperationType::TimeMark:
+ res += 2 * sizeof(ui32);
+ break;
+ case EOperationType::StreamBufAdd:
+ res += 2 * sizeof(ui32);
+ break;
+ }
+
+ return res;
+}
+
+ui32 TSpillMetaRecord::DataSize() {
+ return DataSize_;
+}
+
+ui32 TSpillMetaRecord::Offset() {
+ return Offset_;
+}
+
+ui32 TSpillMetaRecord::DataHash() {
+ return DataHash_;
+}
+
+void TSpillMetaRecord::Pack (TBuffer &buf ) {
+ ui32 size = Size();
+ buf.Resize(size);
+
+ ui32 * bufPtr = (ui32*) buf.Data();
+ *(bufPtr) = RecordHash_;
+ *(bufPtr+1) = Offset_;
+ *(bufPtr+2) = RecordNumber_;
+ *(bufPtr+3) = Meta1_;
+ *(bufPtr+4) = DataSize_;
+ *(bufPtr+5) = DataHash_;
+ ui32 strUi32Size = CalcUi32StrLen(Name_.size());
+ std::copy_n(Name_.data(), Name_.size(), buf.Data() + 6*sizeof(ui32) );
+ *(bufPtr + 6 + strUi32Size ) = size;
+ XXH32_hash_t hash = XXH32( buf.Data() + sizeof(ui32), size - sizeof(ui32), 0);
+ *(bufPtr) = hash;
+ RecordHash_ = hash;
+
+}
+
+void TSpillMetaRecord::Unpack (TBuffer &buf ){
+ ui32 * bufPtr = (ui32*) buf.Data();
+ RecordHash_ = *(bufPtr);
+ Offset_ = *(bufPtr+1);
+ RecordNumber_ = *(bufPtr+2);
+ Meta1_ = *(bufPtr+3);
+ DataSize_ = *(bufPtr+4);
+ DataHash_ = *(bufPtr+5);
+ ui32 nameSize = GetNameSize();
+ Name_.clear();
+ Name_.append( buf.Data() + 6*sizeof(ui32) , nameSize );
+ XXH32_hash_t hash = XXH32( buf.Data() + sizeof(ui32), Size() - sizeof(ui32), 0);
+ if (hash != RecordHash_) {
+ YQL_LOG(ERROR) << "Invalid hash: " << hash << " Record hash: " << RecordHash_ << Endl;
+ }
+}
+
+TString TSpillMetaRecord::AsString() {
+ std::string res;
+ res = "RecordHash: " + std::to_string(RecordHash_) + " |Offset: " + std::to_string(Offset_) + " |RecordNumber: " + std::to_string(RecordNumber_) + " |Meta1: " + std::format("{:#08x}", Meta1_) +
+ " |DataSize: " + std::to_string(DataSize_) + " |DataHash: " + std::to_string(DataHash_) + " |Name: " + Name_;
+
+ return std::move(res);
+
+}
+
+
+void TSpillMetaRecord::SetOpType(EOperationType opType) {
+ ui32 opTypeVal = ((ui32) opType) << 28;
+// Cout << "opTypeVal: " << opTypeVal << Endl;
+ Meta1_ = (opTypeVal | (Meta1_ & (~Meta1OperationBitMask) ));
+}
+
+
+void TSpillMetaRecord::SetNameSize() {
+ ui32 size = Name_.size();
+ Meta1_ = (size | (Meta1_ & (~Meta1OperationStrLen) ));
+
+}
+
+
+void TSpillMetaRecord::ScanForValidSpillRecords(TBuffer& buf, ui32& lastValidOffset, std::vector<TSpillMetaRecord>& records ) {
+
+}
+
+TSpillMetaRecord::TSpillMetaRecord(EOperationType opType, TString& name, ui32 offset, ui32 recordNum, ui32 dataSize, ui32 dataHash ) :
+ Name_(name),
+ Offset_(offset),
+ RecordNumber_(recordNum),
+ DataSize_(dataSize),
+ DataHash_(dataHash)
+{
+ SetOpType(opType);
+ SetNameSize();
+ Bytes_ = Size();
+}
+
+TSpillMetaRecord::TSpillMetaRecord() {};
+
+}
+}
+
diff --git a/ydb/library/yql/core/spilling/storage/storage.h b/ydb/library/yql/core/spilling/storage/storage.h
new file mode 100644
index 00000000000..f9dd97967ec
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/storage.h
@@ -0,0 +1,121 @@
+#pragma once
+
+#include <library/cpp/threading/future/async.h>
+#include <ydb/library/yql/core/spilling/interface/spilling.h>
+#include <util/system/datetime.h>
+
+namespace NYql {
+namespace NSpilling {
+
+class ISpillStorage; // Unified interface to implement spilling based on any supported storage type
+
+// Factory method to create ISpillStorage based on file system for usage. Error reasons are returned in TOperatonResults
+std::pair< THolder<ISpillStorage>, TOperationResults > OpenFileStorageForSpilling(const TFileStorageConfig& config);
+
+const ui32 MagicForFileRecord32 = 0xD0F5F39F; // Magic header number to restore records in case of broken file
+
+// Defines possible operation types
+enum class EOperationType {
+ Add = 1, // Add new temporary object
+ Delete = 2, // Delete temporary object
+ TimeMark = 3, // Add timemark to estimate timing of operations in meta file
+ StreamAdd = 4, // Add new stream sequence
+ StreamDelete = 5, // Deletes stream sequence
+ StreamBufAdd = 6, // Adding new stream buffer
+ SessionDelete = 7, // Deletes session and all objects associated with session
+};
+
+
+// Storage meta record is used to describe particular spill object operation
+class TSpillMetaRecord {
+public:
+ EOperationType GetOpType(); // Returns operation type of the record
+ ui32 GetNameSize(); // Returns size of the record name
+ ui32 Size(); // Total size of record in bytes
+ ui32 DataSize(); // Size of the record Data
+ ui32 Offset(); // Offset of the data in data file
+ ui32 DataHash(); // Returns data hash
+ void Pack (TBuffer& buf ); // Serializes record to buf. After the call Buf is ready to be written to spill meta file
+ void Unpack (TBuffer& buf ); // Restores internal state from buf.
+ TString AsString(); // Returns string representation of SpillMetaRecord
+ void SetDataHash(const ui32 hash) {DataHash_ = hash;}
+ void SetObjId(const ui32 objId) {RecordNumber_ = objId;}
+ // Scans buf for all valid spill meta records and return all recods found in record argument. lastValidOffset - last byte offset of last valid record in buf
+ static void ScanForValidSpillRecords(TBuffer& buf, ui32& lastValidOffset, std::vector<TSpillMetaRecord>& records );
+ TSpillMetaRecord(EOperationType opType, TString& name, ui32 offset, ui32 recordNum, ui32 dataSize, ui32 dataHash );
+ TSpillMetaRecord();
+private:
+ void SetOpType(EOperationType opType = EOperationType::Add); // Defines operation type of the record
+ void SetNameSize(); // Sets str size in bytes in Meta1 record
+ ui32 RecordHash_ = 0; // Hash number to check consistency of record
+ ui32 Offset_ = 0; // Offset of the object in the data file
+ ui32 RecordNumber_ = 0; // Global increasing record number for particular namespace
+ ui32 Meta1_ = 0; // Meta information about the record. See below description of bit masks to unpack info from this field
+ ui32 DataSize_ = 0; // Data size of the buffer in dat file associated with record
+ ui32 DataHash_ = 0; // Hash of buffer in dat file
+ std::string Name_; // Name of the object
+ ui32 Bytes_ = 0; // Total bytes of spill meta record in meta file (28 + size of Name rounded to 4)
+
+};
+
+struct TSpillStreamBufRecord : public TSpillMetaRecord {
+ ui32 StreamId = 0; // Record id of the stream
+ ui32 BufId = 0; // Id of the buffer inside the stream
+};
+
+const ui32 Meta1OperationBitMask = 0xF0000000; // Defines operation type stored in the record
+const ui32 Meta1OperationStrLen = 0x0000FFFF; // Bit mask to extract length of spill object name (maximum 4095 bytes)
+const ui32 Meta1OperationRetention = 0x0FF00000; // Retention period in hours. Value 0xFF (255) means unlimited retention period, it should be deleted manually
+
+
+// Time mark record to estimate in between records addition time
+struct TTimeMarkRecord : public TSpillMetaRecord {
+ ui32 Microseconds1 = 0; // Current Microseconds time since epoch (minor 32 bits)
+ ui32 Microseconds2 = 0; // Current Microseconds time since epoch (major 32 bits)
+};
+
+// Interface for spill file for particular namespace
+class ISpillFile {
+public:
+ virtual TString GetName() = 0; // Returns full name of the file
+ virtual bool IsLocked() = 0; // True when file is locked
+ virtual ui64 Reserve(ui32 size) = 0; // Reserves size bytes for writing, returns file offset
+ virtual void Write(ui32 offset, const char * data, ui32 bytes) = 0;
+ virtual void Seek(ui32 offset) = 0;
+ // Reads up to 1 GB without retrying, returns -1 on error
+ virtual i32 Read(ui32 offset, char* buf, ui32 len) = 0;
+ virtual void Delete() = 0; // Deletes spill file
+ virtual ~ISpillFile() = default;
+};
+
+
+// Interface to work with particular
+class ISpillStorage {
+public:
+
+ virtual ui64 GetCurrSize() = 0; // Returns current size of spill storage
+
+ // Returns full list of namespaces for current spill storage.
+ // All namespaces are on the same level and don't form hierarchy.
+ // Number of namespaces should be reasonable (no more than 1000)
+ // Namespace name should be alphanumerical only.
+ virtual TVector<TString> GetNamespaces() = 0;
+
+ // Returns list of file names for namespace
+ virtual TVector<TString> GetNamespaceFiles(const TString& ns) = 0;
+
+ // Creates file in namespace ns with name fn. File interface ISpillFile is ready for writing and locked in case of success.
+ // Caller should check success with LastOperationResults. reserveStep is a step to increase allocated file storage.
+ virtual THolder<ISpillFile> CreateSpillFile(const TString& ns, const TString& fn, ui32 reserveStep) = 0;
+
+ // Returns last operation results
+ virtual TOperationResults LastOperationResults() = 0;
+
+
+ virtual ~ISpillStorage() = default;
+
+};
+
+
+}
+} \ No newline at end of file
diff --git a/ydb/library/yql/core/spilling/storage/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..649467a4f98
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,78 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-spilling-storage-ut)
+target_compile_options(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage
+)
+target_link_libraries(ydb-library-yql-core-spilling-storage-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ core-spilling-storage
+ yql-utils-log
+)
+target_link_options(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-spilling-storage-ut
+ TEST_TARGET
+ ydb-library-yql-core-spilling-storage-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-core-spilling-storage-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-core-spilling-storage-ut)
diff --git a/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..44f49c0573d
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,81 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-spilling-storage-ut)
+target_compile_options(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage
+)
+target_link_libraries(ydb-library-yql-core-spilling-storage-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ core-spilling-storage
+ yql-utils-log
+)
+target_link_options(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-spilling-storage-ut
+ TEST_TARGET
+ ydb-library-yql-core-spilling-storage-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-core-spilling-storage-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-core-spilling-storage-ut)
diff --git a/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..3d9aeb7cdef
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,83 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-spilling-storage-ut)
+target_compile_options(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage
+)
+target_link_libraries(ydb-library-yql-core-spilling-storage-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ core-spilling-storage
+ yql-utils-log
+)
+target_link_options(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-spilling-storage-ut
+ TEST_TARGET
+ ydb-library-yql-core-spilling-storage-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-core-spilling-storage-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-core-spilling-storage-ut)
diff --git a/ydb/library/yql/core/spilling/storage/ut/CMakeLists.txt b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/spilling/storage/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..0185830a78a
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,71 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-spilling-storage-ut)
+target_compile_options(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage
+)
+target_link_libraries(ydb-library-yql-core-spilling-storage-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ core-spilling-storage
+ yql-utils-log
+)
+target_sources(ydb-library-yql-core-spilling-storage-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-spilling-storage-ut
+ TEST_TARGET
+ ydb-library-yql-core-spilling-storage-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-storage-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-core-spilling-storage-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-core-spilling-storage-ut)
diff --git a/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp b/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp
new file mode 100644
index 00000000000..1c42e106292
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/ut/storage_ut.cpp
@@ -0,0 +1,104 @@
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <chrono>
+#include <iostream>
+#include <cstring>
+#include <vector>
+#include <cassert>
+
+#include <util/system/compiler.h>
+#include <util/stream/null.h>
+#include <util/system/fs.h>
+
+#include <cstdint>
+
+#include <ydb/library/yql/core/spilling/interface/spilling.h>
+#include <ydb/library/yql/core/spilling/storage/storage.h>
+
+namespace NYql {
+namespace NSpilling {
+
+using namespace NSpilling;
+
+constexpr bool IsVerbose = true;
+#define CTEST (IsVerbose ? Cerr : Cnull)
+
+
+Y_UNIT_TEST_SUITE(TSpillingTest) {
+
+Y_UNIT_TEST(TestCreateStorage) {
+ TFileStorageConfig config;
+ config.Path = NFs::CurrentWorkingDirectory();
+ std::pair< THolder<ISpillStorage>, TOperationResults > sp = OpenFileStorageForSpilling(config);
+ CTEST << "Path: " << config.Path << Endl;
+ UNIT_ASSERT(sp.second.Status == EOperationStatus::Success);
+}
+
+
+Y_UNIT_TEST(TestCreateNoStorage) {
+ TFileStorageConfig config;
+ config.Path = "Temp123456";
+ std::pair< THolder<ISpillStorage>, TOperationResults > sp = OpenFileStorageForSpilling(config);
+ CTEST << "Path: " << config.Path << Endl;
+ CTEST << "Error string: " << sp.second.ErrorString << Endl;
+ UNIT_ASSERT(sp.second.Status == EOperationStatus::CannotOpenStorageProxy);
+}
+
+Y_UNIT_TEST(TestNamespaces) {
+ TFileStorageConfig config;
+ config.Path = NFs::CurrentWorkingDirectory();
+ std::pair< THolder<ISpillStorage>, TOperationResults > sp = OpenFileStorageForSpilling(config);
+ TVector<TString> ns = sp.first->GetNamespaces();
+ for ( auto n : ns) {
+ CTEST << "Namespace: " << n << Endl;
+ TVector<TString> nsf = sp.first->GetNamespaceFiles(n);
+ TOperationResults opres = sp.first->LastOperationResults();
+ for ( auto n : nsf) {
+ CTEST << "File: " << n << Endl;
+ }
+ UNIT_ASSERT(opres.Status == EOperationStatus::Success);
+ }
+ UNIT_ASSERT(sp.second.Status == EOperationStatus::Success);
+ UNIT_ASSERT(ns.size() != 0);
+}
+
+
+Y_UNIT_TEST(TestRecordSizes) {
+ size_t s1 = sizeof(TSpillMetaRecord);
+ size_t s2 = sizeof(TTimeMarkRecord);
+ CTEST << "SpillMetaRecord bytes: " << s1 << Endl;
+ CTEST << "TimeMarkRecord bytes: " << s2 << Endl;
+ UNIT_ASSERT(s1+8 == s2);
+}
+
+Y_UNIT_TEST(TestCreateNamespaces) {
+
+ TFileStorageConfig config;
+ config.Path = NFs::CurrentWorkingDirectory();
+ std::pair< THolder<ISpillStorage>, TOperationResults > sp = OpenFileStorageForSpilling(config);
+ TVector<TString> ns = sp.first->GetNamespaces();
+ for ( auto n : ns) {
+ CTEST << "Namespace: " << n << Endl;
+ TVector<TString> nsf = sp.first->GetNamespaceFiles(n);
+ TOperationResults opres = sp.first->LastOperationResults();
+ for ( auto n : nsf) {
+ CTEST << "File: " << n << Endl;
+ }
+ UNIT_ASSERT(opres.Status == EOperationStatus::Success);
+ }
+ THolder<ISpillFile> spf = sp.first->CreateSpillFile(TString("temp1"), TString("ydbspl.1.met"), 4*1024*1024);
+ TOperationResults opres = sp.first->LastOperationResults();
+ UNIT_ASSERT(opres.Status == EOperationStatus::Success);
+ UNIT_ASSERT(spf->IsLocked());
+
+}
+
+
+
+}
+
+}
+}
+
+
diff --git a/ydb/library/yql/core/spilling/storage/ut/ya.make b/ydb/library/yql/core/spilling/storage/ut/ya.make
new file mode 100644
index 00000000000..51f7c47eeb3
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/ut/ya.make
@@ -0,0 +1,32 @@
+UNITTEST_FOR(ydb/library/yql/core/spilling/storage)
+
+FORK_SUBTESTS()
+
+SPLIT_FACTOR(60)
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ TIMEOUT(3600)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCS(
+ storage_ut.cpp
+ )
+
+PEERDIR(
+ ydb/library/yql/utils/log
+)
+
+YQL_LAST_ABI_VERSION()
+
+IF (MKQL_RUNTIME_VERSION)
+ CFLAGS(
+ -DMKQL_RUNTIME_VERSION=$MKQL_RUNTIME_VERSION
+ )
+ENDIF()
+
+END()
diff --git a/ydb/library/yql/core/spilling/storage/ya.make b/ydb/library/yql/core/spilling/storage/ya.make
new file mode 100644
index 00000000000..8c4746a1a39
--- /dev/null
+++ b/ydb/library/yql/core/spilling/storage/ya.make
@@ -0,0 +1,36 @@
+LIBRARY()
+
+SRCS(
+ storage.h
+ storage.cpp
+)
+
+PEERDIR(
+ ydb/library/yql/core/spilling/storage/file_storage
+)
+
+NO_COMPILER_WARNINGS()
+
+IF (NOT MKQL_DISABLE_CODEGEN)
+ PEERDIR(
+ ydb/library/yql/minikql/codegen
+ contrib/libs/llvm12/lib/IR
+ contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
+ contrib/libs/llvm12/lib/Linker
+ contrib/libs/llvm12/lib/Target/X86
+ contrib/libs/llvm12/lib/Target/X86/AsmParser
+ contrib/libs/llvm12/lib/Transforms/IPO
+ )
+ELSE()
+ CFLAGS(
+ -DMKQL_DISABLE_CODEGEN
+ )
+ENDIF()
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/core/spilling/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/core/spilling/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..ba4e853e58f
--- /dev/null
+++ b/ydb/library/yql/core/spilling/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,79 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-spilling-ut)
+target_compile_options(ydb-library-yql-core-spilling-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-spilling-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling
+)
+target_link_libraries(ydb-library-yql-core-spilling-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-core-spilling
+ yql-public-udf
+ udf-service-exception_policy
+)
+target_link_options(ydb-library-yql-core-spilling-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-core-spilling-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/ut/spilling_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-spilling-ut
+ TEST_TARGET
+ ydb-library-yql-core-spilling-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-core-spilling-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-core-spilling-ut)
diff --git a/ydb/library/yql/core/spilling/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/core/spilling/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..0fe44c24967
--- /dev/null
+++ b/ydb/library/yql/core/spilling/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,82 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-spilling-ut)
+target_compile_options(ydb-library-yql-core-spilling-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-spilling-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling
+)
+target_link_libraries(ydb-library-yql-core-spilling-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ yql-core-spilling
+ yql-public-udf
+ udf-service-exception_policy
+)
+target_link_options(ydb-library-yql-core-spilling-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-core-spilling-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/ut/spilling_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-spilling-ut
+ TEST_TARGET
+ ydb-library-yql-core-spilling-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-core-spilling-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-core-spilling-ut)
diff --git a/ydb/library/yql/core/spilling/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/core/spilling/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..33abfe68c08
--- /dev/null
+++ b/ydb/library/yql/core/spilling/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,84 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-spilling-ut)
+target_compile_options(ydb-library-yql-core-spilling-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-spilling-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling
+)
+target_link_libraries(ydb-library-yql-core-spilling-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-core-spilling
+ yql-public-udf
+ udf-service-exception_policy
+)
+target_link_options(ydb-library-yql-core-spilling-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-core-spilling-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/ut/spilling_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-spilling-ut
+ TEST_TARGET
+ ydb-library-yql-core-spilling-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-core-spilling-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-core-spilling-ut)
diff --git a/ydb/library/yql/core/spilling/ut/CMakeLists.txt b/ydb/library/yql/core/spilling/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/core/spilling/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/core/spilling/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/core/spilling/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..6d5ce50c09b
--- /dev/null
+++ b/ydb/library/yql/core/spilling/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,72 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-core-spilling-ut)
+target_compile_options(ydb-library-yql-core-spilling-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-core-spilling-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling
+)
+target_link_libraries(ydb-library-yql-core-spilling-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-core-spilling
+ yql-public-udf
+ udf-service-exception_policy
+)
+target_sources(ydb-library-yql-core-spilling-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/core/spilling/ut/spilling_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-core-spilling-ut
+ TEST_TARGET
+ ydb-library-yql-core-spilling-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-core-spilling-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-core-spilling-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-core-spilling-ut)
diff --git a/ydb/library/yql/core/spilling/ut/spilling_ut.cpp b/ydb/library/yql/core/spilling/ut/spilling_ut.cpp
new file mode 100644
index 00000000000..d189a503a14
--- /dev/null
+++ b/ydb/library/yql/core/spilling/ut/spilling_ut.cpp
@@ -0,0 +1,368 @@
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <chrono>
+#include <iostream>
+#include <cstring>
+#include <vector>
+#include <cassert>
+#include <thread>
+
+#include <util/system/fs.h>
+#include <util/system/compiler.h>
+#include <util/stream/null.h>
+#include <util/system/mem_info.h>
+
+#include <cstdint>
+
+#include <ydb/library/yql/core/spilling/interface/spilling.h>
+#include <ydb/library/yql/core/spilling/storage/storage.h>
+
+namespace NYql {
+namespace NSpilling {
+
+using namespace NSpilling;
+using namespace std::chrono_literals;
+
+constexpr bool IsVerbose = true;
+#define CTEST (IsVerbose ? Cerr : Cnull)
+
+
+Y_UNIT_TEST_SUITE(TYDBLibrarySpillingTest) {
+
+ Y_UNIT_TEST(TestCreateProxy) {
+ TFileStorageConfig config;
+ config.Path = NFs::CurrentWorkingDirectory();
+ TTempStorageExecutionPolicy policy;
+ std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy );
+ CTEST << "Status: " << ui32( tproxy.second.Status ) << Endl;
+ UNIT_ASSERT(tproxy.second.Status == EOperationStatus::Success);
+ }
+
+ Y_UNIT_TEST(TestCreateSession) {
+ TFileStorageConfig config;
+ config.Path = NFs::CurrentWorkingDirectory();
+ TTempStorageExecutionPolicy policy;
+ std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy );
+ THolder<ISession> session = tproxy.first->CreateSession();
+ UNIT_ASSERT(session != nullptr);
+ }
+
+ Y_UNIT_TEST(TestSave) {
+ TFileStorageConfig config;
+ config.Path = NFs::CurrentWorkingDirectory();
+ TTempStorageExecutionPolicy policy;
+ std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy );
+ THolder<ISession> session = tproxy.first->CreateSession();
+ NThreading::TFuture<TOperationResults> ftr;
+ const ui32 bufSize = 1024*1024;
+ const ui32 iters = 1000;
+ std::vector<TBuffer> buffers, buffers1;
+
+ NMemInfo::TMemInfo mi = NMemInfo::GetMemInfo();
+ CTEST << "Mem usage before buffs prepare (MB): " << mi.RSS / (1024 * 1024) << Endl;
+ std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
+ for (ui32 i = 0; i < iters; i++) {
+ TBuffer buf(bufSize);
+ buf.Resize(bufSize);
+ memset(buf.Data(), i+1, bufSize/sizeof(int) - 1 );
+ buffers1.emplace_back(std::move(buf));
+ }
+ mi = NMemInfo::GetMemInfo();
+ CTEST << "Mem usage after buffs prepare (MB): " << mi.RSS / (1024 * 1024) << Endl;
+ std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
+ ui64 execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
+ CTEST << "Execution time for " << iters << " prepare buf calls (microseconds): " << execTime << Endl;
+
+ begin = std::chrono::steady_clock::now();
+ for (ui32 i = 0; i < iters; i++) {
+ buffers.emplace_back(std::move(buffers1[i]));
+ }
+ end = std::chrono::steady_clock::now();
+ execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
+ CTEST << "Execution time for " << iters << " moving buffers: (microseconds): " << execTime << Endl;
+
+
+ begin = std::chrono::steady_clock::now();
+ for (ui32 i = 0; i < iters; i++) {
+ ftr = session->Save(TString("Test"), TString("test" + std::to_string(i)), std::move(buffers[i]));
+ }
+ end = std::chrono::steady_clock::now();
+ execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
+ CTEST << "Execution time for " << iters << " save calls (microseconds): " << execTime << Endl;
+ CTEST << "Per one call (microseconds): " << execTime/iters << Endl;
+
+ mi = NMemInfo::GetMemInfo();
+ CTEST << "Mem usage after buffs save (MB): " << mi.RSS / (1024 * 1024) << Endl;
+
+ TSessionDataStat dstat = session->GetSessionDataStat();
+ CTEST << "Session total data: " << dstat.Provided << Endl;
+ CTEST << "Session spilled data: " << dstat.Spilled << Endl;
+ CTEST << "Session in memory data: " << dstat.InMemory << Endl;
+
+ std::this_thread::sleep_for(5000ms);
+
+ dstat = session->GetSessionDataStat();
+ CTEST << "Session spilled data after sleep: " << dstat.Spilled << Endl;
+ CTEST << "Session loaded from memory data: " << dstat.LoadedFromMemory << Endl;
+ CTEST << "Session in memory data: " << dstat.InMemory << Endl;
+
+ mi = NMemInfo::GetMemInfo();
+ CTEST << "Mem usage after buffs writing (MB): " << mi.RSS / (1024 * 1024) << Endl;
+
+ UNIT_ASSERT(ftr.Initialized());
+ }
+
+ Y_UNIT_TEST(TestLoad) {
+ TFileStorageConfig config;
+ config.Path = NFs::CurrentWorkingDirectory();
+ TTempStorageExecutionPolicy policy;
+ std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy );
+ THolder<ISession> session = tproxy.first->CreateSession();
+
+ NThreading::TFuture<TOperationResults> ftr;
+ std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
+ const ui32 bufSize = 1024*1024;
+ const ui32 iters = 1000;
+ for (ui32 i = 0; i < iters; i++) {
+ TBuffer buf(bufSize);
+ buf.Resize(bufSize);
+ memset(buf.Data(), i+1, bufSize/sizeof(int) - 1 );
+ ftr = session->Save(TString("Test"), TString("test" + std::to_string(i)), std::move(buf));
+ }
+ std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
+ ui64 execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
+ CTEST << "Execution time for " << iters << " save calls (microseconds): " << execTime << Endl;
+ CTEST << "Per one call (microseconds): " << execTime/iters << Endl;
+
+// std::this_thread::sleep_for(3000ms);
+
+ begin = std::chrono::steady_clock::now();
+ NThreading::TFuture<TLoadOperationResults> ftrl;
+ for (ui32 i = 0; i < iters; i++) {
+ ftrl = session->Load(TString("Test"), TString("test" + std::to_string(i)));
+ }
+ end = std::chrono::steady_clock::now();
+ execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
+ CTEST << "Execution time for " << iters << " load calls (microseconds): " << execTime << Endl;
+ CTEST << "Per one call (microseconds): " << execTime/iters << Endl;
+
+ TSessionDataStat dstat = session->GetSessionDataStat();
+ CTEST << "Session total data: " << dstat.Provided << Endl;
+ CTEST << "Session spilled data: " << dstat.Spilled << Endl;
+ CTEST << "Session loaded from memory data: " << dstat.LoadedFromMemory << Endl;
+ CTEST << "Session in memory data: " << dstat.InMemory << Endl;
+
+ UNIT_ASSERT(ftrl.Initialized());
+ }
+
+ Y_UNIT_TEST(TestSetOp) {
+ TString name{"Test"};
+ TSpillMetaRecord mr{EOperationType::Add, name, 0, 0, 1, 2};
+ auto res = mr.GetOpType();
+ CTEST << "OpType: " << (ui32) res << Endl;
+ CTEST << mr.AsString() << Endl;
+ UNIT_ASSERT(res == EOperationType::Add);
+
+ }
+
+ Y_UNIT_TEST(TestSpillMetaPack) {
+ TString name{"Test1"};
+ TSpillMetaRecord mr{EOperationType::Add, name, 1, 2, 3, 4};
+ auto res = mr.GetOpType();
+ CTEST << mr.AsString() << Endl;
+ TBuffer buf;
+ mr.Pack(buf);
+ CTEST << "Buf size: " << buf.Size() << Endl;
+
+ TString name1{"Test22222"};
+ TSpillMetaRecord mr1{EOperationType::Add, name1, 2, 3, 5, 6};
+ CTEST << mr1.AsString() << Endl;
+
+ mr1.Unpack(buf);
+ CTEST << mr1.AsString() << Endl;
+
+ UNIT_ASSERT(res == EOperationType::Add);
+
+ }
+
+ Y_UNIT_TEST(TestSpillSaveLoad) {
+ TFileStorageConfig config;
+ config.Path = NFs::CurrentWorkingDirectory();
+ TTempStorageExecutionPolicy policy;
+ std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy );
+ THolder<ISession> session = tproxy.first->CreateSession();
+
+ NThreading::TFuture<TOperationResults> ftr;
+ std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
+ const ui32 bufSize = 1024*1024;
+ const ui32 iters = 1000;
+ for (ui32 i = 0; i < iters; i++) {
+ TBuffer buf(bufSize);
+ buf.Resize(bufSize);
+ memset(buf.Data(), i+1, bufSize/sizeof(int) - 1 );
+ ftr = session->Save(TString("Test"), TString("test" + std::to_string(i)), std::move(buf));
+ }
+ std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
+ ui64 execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
+ CTEST << "Execution time for " << iters << " save calls (microseconds): " << execTime << Endl;
+ CTEST << "Per one call (microseconds): " << execTime/iters << Endl;
+
+ std::this_thread::sleep_for(3000ms);
+
+ begin = std::chrono::steady_clock::now();
+ NThreading::TFuture<TLoadOperationResults> ftrl;
+ for (ui32 i = 0; i < iters; i++) {
+ ftrl = session->Load(TString("Test"), TString("test" + std::to_string(i)));
+ }
+ end = std::chrono::steady_clock::now();
+ execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
+ CTEST << "Execution time for " << iters << " load calls (microseconds): " << execTime << Endl;
+ CTEST << "Per one call (microseconds): " << execTime/iters << Endl;
+
+ std::this_thread::sleep_for(6000ms);
+
+ TSessionDataStat dstat = session->GetSessionDataStat();
+ CTEST << "Session total data: " << dstat.Provided << Endl;
+ CTEST << "Session spilled data: " << dstat.Spilled << Endl;
+ CTEST << "Session loaded from memory data: " << dstat.LoadedFromMemory << Endl;
+ CTEST << "Session loaded from storage data: " << dstat.LoadedFromStorage << Endl;
+ CTEST << "Session in memory data: " << dstat.InMemory << Endl;
+
+ UNIT_ASSERT(ftrl.Initialized());
+
+ }
+
+ Y_UNIT_TEST(TestGarbageCollection) {
+ TFileStorageConfig config;
+ config.Path = NFs::CurrentWorkingDirectory();
+ TTempStorageExecutionPolicy policy;
+ std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy );
+ for (ui32 sessNum = 0; sessNum < 10; sessNum++) {
+ THolder<ISession> session = tproxy.first->CreateSession();
+
+ NThreading::TFuture<TOperationResults> ftr;
+ std::chrono::steady_clock::time_point begin =
+ std::chrono::steady_clock::now();
+ const ui32 bufSize = 1024 * 1024;
+ const ui32 iters = 1000;
+ for (ui32 i = 0; i < iters; i++) {
+ TBuffer buf(bufSize);
+ buf.Resize(bufSize);
+ memset(buf.Data(), i + 1, bufSize / sizeof(int) - 1);
+ ftr = session->Save(TString("Test"),
+ TString("test" + std::to_string(i)),
+ std::move(buf));
+ }
+ std::chrono::steady_clock::time_point end =
+ std::chrono::steady_clock::now();
+ ui64 execTime =
+ std::chrono::duration_cast<std::chrono::microseconds>(end -
+ begin)
+ .count();
+ CTEST << "Execution time for " << iters
+ << " save calls (microseconds): " << execTime << Endl;
+ CTEST << "Per one call (microseconds): " << execTime / iters
+ << Endl;
+
+ std::this_thread::sleep_for(3000ms);
+
+ TSessionDataStat dstat = session->GetSessionDataStat();
+ CTEST << "Session total data after save: " << dstat.Provided << Endl;
+ CTEST << "Session spilled data after save: " << dstat.Spilled << Endl;
+ CTEST << "Session loaded from memory data after save : "
+ << dstat.LoadedFromMemory << Endl;
+ CTEST << "Session loaded from storage data after save: "
+ << dstat.LoadedFromStorage << Endl;
+ CTEST << "Session in memory data after save: " << dstat.InMemory << Endl;
+
+
+ begin = std::chrono::steady_clock::now();
+ NThreading::TFuture<TLoadOperationResults> ftrl;
+ for (ui32 i = 0; i < iters; i++) {
+ ftrl = session->Load(TString("Test"),
+ TString("test" + std::to_string(i)));
+ }
+ end = std::chrono::steady_clock::now();
+ execTime = std::chrono::duration_cast<std::chrono::microseconds>(
+ end - begin)
+ .count();
+ CTEST << "Execution time for " << iters
+ << " load calls (microseconds): " << execTime << Endl;
+ CTEST << "Per one call (microseconds): " << execTime / iters
+ << Endl;
+
+ std::this_thread::sleep_for(6000ms);
+
+ dstat = session->GetSessionDataStat();
+ CTEST << "Session total data: " << dstat.Provided << Endl;
+ CTEST << "Session spilled data: " << dstat.Spilled << Endl;
+ CTEST << "Session loaded from memory data: "
+ << dstat.LoadedFromMemory << Endl;
+ CTEST << "Session loaded from storage data: "
+ << dstat.LoadedFromStorage << Endl;
+ CTEST << "Session in memory data: " << dstat.InMemory << Endl;
+
+ UNIT_ASSERT(ftrl.Initialized());
+ }
+ }
+
+
+ Y_UNIT_TEST(TestSpillStreamSaveLoad) {
+ TFileStorageConfig config;
+ config.Path = NFs::CurrentWorkingDirectory();
+ TTempStorageExecutionPolicy policy;
+ std::pair < THolder<ITempStorageProxy>, TOperationResults> tproxy = CreateFileStorageProxy(config, policy );
+ THolder<ISession> session = tproxy.first->CreateSession();
+
+ NThreading::TFuture<TOperationResults> ftr;
+ std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
+ const ui32 bufSize = 1024*1024;
+ const ui32 iters = 1000;
+ auto res = session->OpenStream(TString("Test"), TString("Stream1"));
+ auto st1 = std::move(res.first);
+ CTEST << "Stream size before save: " << st1->Size() << Endl;
+ for (ui32 i = 0; i < iters; i++) {
+ TBuffer buf(bufSize);
+ buf.Resize(bufSize);
+ memset(buf.Data(), i+1, bufSize/sizeof(int) - 1 );
+ ftr = st1->Save(std::move(buf));
+ }
+ CTEST << "Stream size after save: " << st1->Size() << Endl;
+ std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
+ ui64 execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
+ CTEST << "Execution time for " << iters << " save calls (microseconds): " << execTime << Endl;
+ CTEST << "Per one call (microseconds): " << execTime/iters << Endl;
+
+ std::this_thread::sleep_for(3000ms);
+
+ begin = std::chrono::steady_clock::now();
+ NThreading::TFuture<TLoadOperationResults> ftrl;
+ for (ui32 i = 0; i < iters; i++) {
+ ftrl = st1->Load(i);
+ }
+ end = std::chrono::steady_clock::now();
+ execTime = std::chrono::duration_cast<std::chrono::microseconds>(end - begin).count();
+ CTEST << "Execution time for " << iters << " load calls (microseconds): " << execTime << Endl;
+ CTEST << "Per one call (microseconds): " << execTime/iters << Endl;
+
+ std::this_thread::sleep_for(6000ms);
+
+ TSessionDataStat dstat = session->GetSessionDataStat();
+ CTEST << "Session total data: " << dstat.Provided << Endl;
+ CTEST << "Session spilled data: " << dstat.Spilled << Endl;
+ CTEST << "Session loaded from memory data: " << dstat.LoadedFromMemory << Endl;
+ CTEST << "Session loaded from storage data: " << dstat.LoadedFromStorage << Endl;
+ CTEST << "Session in memory data: " << dstat.InMemory << Endl;
+
+ UNIT_ASSERT(ftrl.Initialized());
+
+ }
+
+
+
+}
+
+}
+
+}
diff --git a/ydb/library/yql/core/spilling/ut/ya.make b/ydb/library/yql/core/spilling/ut/ya.make
new file mode 100644
index 00000000000..7c97cbed1b8
--- /dev/null
+++ b/ydb/library/yql/core/spilling/ut/ya.make
@@ -0,0 +1,35 @@
+UNITTEST_FOR(ydb/library/yql/core/spilling)
+
+FORK_SUBTESTS()
+
+SPLIT_FACTOR(60)
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ TIMEOUT(3600)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCS(
+ spilling_ut.cpp
+ )
+
+PEERDIR(
+ ydb/library/yql/public/udf
+ ydb/library/yql/public/udf/service/exception_policy
+)
+
+YQL_LAST_ABI_VERSION()
+
+IF (MKQL_RUNTIME_VERSION)
+ CFLAGS(
+ -DMKQL_RUNTIME_VERSION=$MKQL_RUNTIME_VERSION
+ )
+ENDIF()
+
+REQUIREMENTS(ram:10)
+
+END()
diff --git a/ydb/library/yql/core/spilling/ya.make b/ydb/library/yql/core/spilling/ya.make
new file mode 100644
index 00000000000..af3aa1592c6
--- /dev/null
+++ b/ydb/library/yql/core/spilling/ya.make
@@ -0,0 +1,52 @@
+LIBRARY()
+
+SRCS(
+ spilling_imp.cpp
+ spilling_imp.h
+ namespaces_list.h
+ namespaces_list.cpp
+ namespace_cache.h
+ namespace_cache.cpp
+ interface/spilling.h
+ storage/file_storage/file_storage.cpp
+ storage/storage.h
+ storage/storage.cpp
+)
+
+PEERDIR(
+ contrib/libs/apache/arrow
+ ydb/library/binary_json
+ ydb/library/yql/utils
+ ydb/library/yql/utils/log
+ ydb/library/yql/core/spilling/storage
+)
+
+NO_COMPILER_WARNINGS()
+
+IF (NOT MKQL_DISABLE_CODEGEN)
+ PEERDIR(
+ ydb/library/yql/minikql/codegen
+ contrib/libs/llvm12/lib/IR
+ contrib/libs/llvm12/lib/ExecutionEngine/MCJIT
+ contrib/libs/llvm12/lib/Linker
+ contrib/libs/llvm12/lib/Target/X86
+ contrib/libs/llvm12/lib/Target/X86/AsmParser
+ contrib/libs/llvm12/lib/Transforms/IPO
+ )
+ELSE()
+ CFLAGS(
+ -DMKQL_DISABLE_CODEGEN
+ )
+ENDIF()
+
+YQL_LAST_ABI_VERSION()
+
+END()
+
+RECURSE(
+ storage
+)
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/core/url_lister/ya.make b/ydb/library/yql/core/url_lister/ya.make
index 2caf6cf6f0c..b11cf9dda46 100644
--- a/ydb/library/yql/core/url_lister/ya.make
+++ b/ydb/library/yql/core/url_lister/ya.make
@@ -11,3 +11,7 @@ PEERDIR(
)
END()
+
+RECURSE(
+ interface
+)
diff --git a/ydb/library/yql/core/url_preprocessing/ya.make b/ydb/library/yql/core/url_preprocessing/ya.make
index c062ee36f2f..0449776e356 100644
--- a/ydb/library/yql/core/url_preprocessing/ya.make
+++ b/ydb/library/yql/core/url_preprocessing/ya.make
@@ -14,3 +14,7 @@ PEERDIR(
)
END()
+
+RECURSE(
+ interface
+)
diff --git a/ydb/library/yql/core/ya.make b/ydb/library/yql/core/ya.make
index ec7b793d991..4fa07131448 100644
--- a/ydb/library/yql/core/ya.make
+++ b/ydb/library/yql/core/ya.make
@@ -92,4 +92,25 @@ YQL_LAST_ABI_VERSION()
END()
+RECURSE(
+ arrow_kernels
+ cbo
+ common_opt
+ credentials
+ expr_nodes
+ expr_nodes_gen
+ extract_predicate
+ facade
+ file_storage
+ issue
+ peephole_opt
+ services
+ spilling
+ sql_types
+ type_ann
+ url_lister
+ url_preprocessing
+ user_data
+)
+
RECURSE_FOR_TESTS(ut)
diff --git a/ydb/library/yql/minikql/arrow/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/arrow/CMakeLists.darwin-x86_64.txt
index 5bb17a45a42..b40197fd49c 100644
--- a/ydb/library/yql/minikql/arrow/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/minikql/arrow/CMakeLists.darwin-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-arrow)
target_compile_options(yql-minikql-arrow PRIVATE
diff --git a/ydb/library/yql/minikql/arrow/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/arrow/CMakeLists.linux-aarch64.txt
index 73213515674..38d86148521 100644
--- a/ydb/library/yql/minikql/arrow/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/minikql/arrow/CMakeLists.linux-aarch64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-arrow)
target_compile_options(yql-minikql-arrow PRIVATE
diff --git a/ydb/library/yql/minikql/arrow/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/arrow/CMakeLists.linux-x86_64.txt
index 73213515674..38d86148521 100644
--- a/ydb/library/yql/minikql/arrow/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/minikql/arrow/CMakeLists.linux-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-arrow)
target_compile_options(yql-minikql-arrow PRIVATE
diff --git a/ydb/library/yql/minikql/arrow/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/arrow/CMakeLists.windows-x86_64.txt
index 5bb17a45a42..b40197fd49c 100644
--- a/ydb/library/yql/minikql/arrow/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/minikql/arrow/CMakeLists.windows-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-arrow)
target_compile_options(yql-minikql-arrow PRIVATE
diff --git a/ydb/library/yql/minikql/arrow/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/arrow/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..245ab8c1813
--- /dev/null
+++ b/ydb/library/yql/minikql/arrow/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,81 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-arrow-ut)
+target_compile_options(ydb-library-yql-minikql-arrow-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-arrow-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow
+)
+target_link_libraries(ydb-library-yql-minikql-arrow-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-arrow
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ minikql-invoke_builtins-llvm
+)
+target_link_options(ydb-library-yql-minikql-arrow-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-minikql-arrow-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow/mkql_functions_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-arrow-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-arrow-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-arrow-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-arrow-ut)
diff --git a/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..719f3857393
--- /dev/null
+++ b/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,84 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-arrow-ut)
+target_compile_options(ydb-library-yql-minikql-arrow-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-arrow-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow
+)
+target_link_libraries(ydb-library-yql-minikql-arrow-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ yql-minikql-arrow
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ minikql-invoke_builtins-llvm
+)
+target_link_options(ydb-library-yql-minikql-arrow-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-arrow-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow/mkql_functions_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-arrow-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-arrow-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-arrow-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-minikql-arrow-ut)
diff --git a/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..d5e533c7bd5
--- /dev/null
+++ b/ydb/library/yql/minikql/arrow/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,86 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-arrow-ut)
+target_compile_options(ydb-library-yql-minikql-arrow-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-arrow-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow
+)
+target_link_libraries(ydb-library-yql-minikql-arrow-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-arrow
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ minikql-invoke_builtins-llvm
+)
+target_link_options(ydb-library-yql-minikql-arrow-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-arrow-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow/mkql_functions_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-arrow-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-arrow-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-arrow-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-minikql-arrow-ut)
diff --git a/ydb/library/yql/minikql/arrow/ut/CMakeLists.txt b/ydb/library/yql/minikql/arrow/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/arrow/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/arrow/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/arrow/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..55cf43ffb11
--- /dev/null
+++ b/ydb/library/yql/minikql/arrow/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,74 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-arrow-ut)
+target_compile_options(ydb-library-yql-minikql-arrow-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-arrow-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow
+)
+target_link_libraries(ydb-library-yql-minikql-arrow-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-arrow
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ minikql-invoke_builtins-llvm
+)
+target_sources(ydb-library-yql-minikql-arrow-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/arrow/mkql_functions_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-arrow-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-arrow-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-arrow-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-arrow-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-arrow-ut)
diff --git a/ydb/library/yql/minikql/codegen/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/codegen/CMakeLists.darwin-x86_64.txt
index cf7af4749a9..377a730d286 100644
--- a/ydb/library/yql/minikql/codegen/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/minikql/codegen/CMakeLists.darwin-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-codegen)
target_compile_options(yql-minikql-codegen PRIVATE
diff --git a/ydb/library/yql/minikql/codegen/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/codegen/CMakeLists.linux-aarch64.txt
index 49938a8c462..6a23884af5a 100644
--- a/ydb/library/yql/minikql/codegen/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/minikql/codegen/CMakeLists.linux-aarch64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-codegen)
target_compile_options(yql-minikql-codegen PRIVATE
diff --git a/ydb/library/yql/minikql/codegen/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/codegen/CMakeLists.linux-x86_64.txt
index 49938a8c462..6a23884af5a 100644
--- a/ydb/library/yql/minikql/codegen/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/minikql/codegen/CMakeLists.linux-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-codegen)
target_compile_options(yql-minikql-codegen PRIVATE
diff --git a/ydb/library/yql/minikql/codegen/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/codegen/CMakeLists.windows-x86_64.txt
index fb5f429c0f1..7fdb97049bc 100644
--- a/ydb/library/yql/minikql/codegen/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/minikql/codegen/CMakeLists.windows-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-codegen)
target_compile_options(yql-minikql-codegen PRIVATE
diff --git a/ydb/library/yql/minikql/codegen/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/codegen/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..fb548f5eeb8
--- /dev/null
+++ b/ydb/library/yql/minikql/codegen/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,157 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_rescompiler_bin
+ TOOL_rescompiler_dependency
+ tools/rescompiler/bin
+ rescompiler
+)
+
+add_executable(ydb-library-yql-minikql-codegen-ut)
+target_compile_options(ydb-library-yql-minikql-codegen-ut PRIVATE
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_include_directories(ydb-library-yql-minikql-codegen-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen
+)
+target_link_libraries(ydb-library-yql-minikql-codegen-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-codegen
+ library-cpp-resource
+)
+target_link_options(ydb-library-yql-minikql-codegen-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-minikql-codegen-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/codegen_ut.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-codegen-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-codegen-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+add_custom_command(
+ OUTPUT
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ DEPENDS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ COMMAND
+ ${LLVMOPT}
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ -o
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ -O2
+ -globalopt
+ -globaldce
+ -internalize
+ -internalize-public-api-list=fib#sum_sqr#sum_sqr2#sum_sqr_128#sum_sqr_128_ir#str_size
+)
+add_custom_command(
+ OUTPUT
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ DEPENDS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc
+ COMMAND
+ ${LLVMLINK}
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc
+ -o
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+resources(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp
+ INPUTS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ KEYS
+ /llvm_bc/Funcs
+)
+target_allocator(ydb-library-yql-minikql-codegen-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-codegen-ut)
diff --git a/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..32a6487c08f
--- /dev/null
+++ b/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,160 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_rescompiler_bin
+ TOOL_rescompiler_dependency
+ tools/rescompiler/bin
+ rescompiler
+)
+
+add_executable(ydb-library-yql-minikql-codegen-ut)
+target_compile_options(ydb-library-yql-minikql-codegen-ut PRIVATE
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_include_directories(ydb-library-yql-minikql-codegen-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen
+)
+target_link_libraries(ydb-library-yql-minikql-codegen-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ yql-minikql-codegen
+ library-cpp-resource
+)
+target_link_options(ydb-library-yql-minikql-codegen-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-codegen-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/codegen_ut.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-codegen-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-codegen-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+add_custom_command(
+ OUTPUT
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ DEPENDS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ COMMAND
+ ${LLVMOPT}
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ -o
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ -O2
+ -globalopt
+ -globaldce
+ -internalize
+ -internalize-public-api-list=fib#sum_sqr#sum_sqr2#sum_sqr_128#sum_sqr_128_ir#str_size
+)
+add_custom_command(
+ OUTPUT
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ DEPENDS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc
+ COMMAND
+ ${LLVMLINK}
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc
+ -o
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+resources(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp
+ INPUTS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ KEYS
+ /llvm_bc/Funcs
+)
+target_allocator(ydb-library-yql-minikql-codegen-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-minikql-codegen-ut)
diff --git a/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..d3a3bee4e27
--- /dev/null
+++ b/ydb/library/yql/minikql/codegen/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,162 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_rescompiler_bin
+ TOOL_rescompiler_dependency
+ tools/rescompiler/bin
+ rescompiler
+)
+
+add_executable(ydb-library-yql-minikql-codegen-ut)
+target_compile_options(ydb-library-yql-minikql-codegen-ut PRIVATE
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_include_directories(ydb-library-yql-minikql-codegen-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen
+)
+target_link_libraries(ydb-library-yql-minikql-codegen-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-codegen
+ library-cpp-resource
+)
+target_link_options(ydb-library-yql-minikql-codegen-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-codegen-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/codegen_ut.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-codegen-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-codegen-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+add_custom_command(
+ OUTPUT
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ DEPENDS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ COMMAND
+ ${LLVMOPT}
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ -o
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ -O2
+ -globalopt
+ -globaldce
+ -internalize
+ -internalize-public-api-list=fib#sum_sqr#sum_sqr2#sum_sqr_128#sum_sqr_128_ir#str_size
+)
+add_custom_command(
+ OUTPUT
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ DEPENDS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc
+ COMMAND
+ ${LLVMLINK}
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.ll.bc
+ -o
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+resources(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp
+ INPUTS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ KEYS
+ /llvm_bc/Funcs
+)
+target_allocator(ydb-library-yql-minikql-codegen-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-minikql-codegen-ut)
diff --git a/ydb/library/yql/minikql/codegen/ut/CMakeLists.txt b/ydb/library/yql/minikql/codegen/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/codegen/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/codegen/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/codegen/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..2524c3f4efd
--- /dev/null
+++ b/ydb/library/yql/minikql/codegen/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,150 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+get_built_tool_path(
+ TOOL_rescompiler_bin
+ TOOL_rescompiler_dependency
+ tools/rescompiler/bin
+ rescompiler
+)
+
+add_executable(ydb-library-yql-minikql-codegen-ut)
+target_compile_options(ydb-library-yql-minikql-codegen-ut PRIVATE
+ $<IF:$<CXX_COMPILER_ID:MSVC>,,-Wno-everything>
+)
+target_include_directories(ydb-library-yql-minikql-codegen-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen
+)
+target_link_libraries(ydb-library-yql-minikql-codegen-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-codegen
+ library-cpp-resource
+)
+target_sources(ydb-library-yql-minikql-codegen-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/codegen_ut.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-codegen-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-codegen-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-codegen-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+add_custom_command(
+ OUTPUT
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ DEPENDS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ COMMAND
+ ${LLVMOPT}
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ -o
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ -O2
+ -globalopt
+ -globaldce
+ -internalize
+ -internalize-public-api-list=fib#sum_sqr#sum_sqr2#sum_sqr_128#sum_sqr_128_ir#str_size
+)
+add_custom_command(
+ OUTPUT
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+ DEPENDS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit_win.ll.bc
+ COMMAND
+ ${LLVMLINK}
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit_win.ll.bc
+ -o
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_merged.bc
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/fib.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/sum_sqr2.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/str.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+llvm_compile_cxx(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/128_bit.cpp.bc
+ ${CLANGPLUSPLUS}
+ -Wno-unknown-warning-option
+ -emit-llvm
+)
+resources(ydb-library-yql-minikql-codegen-ut
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/925fda4cf8bf8097eb12d1681cec1fd0.cpp
+ INPUTS
+ ${CMAKE_BINARY_DIR}/ydb/library/yql/minikql/codegen/ut/Funcs_optimized.bc
+ KEYS
+ /llvm_bc/Funcs
+)
+target_allocator(ydb-library-yql-minikql-codegen-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-codegen-ut)
diff --git a/ydb/library/yql/minikql/comp_nodes/CMakeLists.txt b/ydb/library/yql/minikql/comp_nodes/CMakeLists.txt
index e519707fa41..4716de1ef37 100644
--- a/ydb/library/yql/minikql/comp_nodes/CMakeLists.txt
+++ b/ydb/library/yql/minikql/comp_nodes/CMakeLists.txt
@@ -7,3 +7,5 @@
add_subdirectory(llvm)
+add_subdirectory(no_llvm)
+add_subdirectory(ut)
diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..f10e5ccc395
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,157 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-comp_nodes-no_llvm)
+target_compile_options(minikql-comp_nodes-no_llvm PRIVATE
+ -mprfchw
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-comp_nodes-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-comp_nodes-no_llvm PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ ydb-library-binary_json
+ library-yql-minikql
+ yql-minikql-arrow
+ public-udf-arrow
+ parser-pg_wrapper-interface
+ library-yql-utils
+ cpp-actors-core
+ minikql-invoke_builtins-no_llvm
+)
+target_sources(minikql-comp_nodes-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_addmember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_aggrcount.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_append.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_count.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_minmax.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_some.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_sum.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_coalesce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_extend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_if.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_just.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_compress.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_func.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_skiptake.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_top.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_blocks.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain1_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_check_args.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_coalesce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_collect.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_contains.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_div.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mod.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mul.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_discard.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_element.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ensure.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_enumerate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_exists.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_extend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flow.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold1.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_frombytes.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromyson.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_guess.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hasitems.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_heap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_if.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_invoke.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterable.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lazy_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_length.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_llvm_base.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lookup.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_mapnext.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_nop.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_now.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_pickle.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_prepend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_random.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reduce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_replicate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reverse.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_rh_hash.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_scalar_apply.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_size.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_source.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_state.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_take.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_timezone.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tobytes.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_toindexdict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tooptional.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_unwrap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_way.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_weakmember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_withcontext.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_zip.cpp
+)
diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..96b0c048ea6
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,158 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-comp_nodes-no_llvm)
+target_compile_options(minikql-comp_nodes-no_llvm PRIVATE
+ -mprfchw
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-comp_nodes-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-comp_nodes-no_llvm PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ ydb-library-binary_json
+ library-yql-minikql
+ yql-minikql-arrow
+ public-udf-arrow
+ parser-pg_wrapper-interface
+ library-yql-utils
+ cpp-actors-core
+ minikql-invoke_builtins-no_llvm
+)
+target_sources(minikql-comp_nodes-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_addmember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_aggrcount.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_append.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_count.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_minmax.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_some.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_sum.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_coalesce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_extend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_if.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_just.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_compress.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_func.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_skiptake.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_top.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_blocks.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain1_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_check_args.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_coalesce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_collect.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_contains.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_div.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mod.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mul.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_discard.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_element.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ensure.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_enumerate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_exists.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_extend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flow.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold1.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_frombytes.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromyson.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_guess.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hasitems.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_heap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_if.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_invoke.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterable.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lazy_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_length.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_llvm_base.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lookup.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_mapnext.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_nop.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_now.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_pickle.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_prepend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_random.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reduce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_replicate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reverse.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_rh_hash.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_scalar_apply.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_size.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_source.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_state.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_take.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_timezone.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tobytes.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_toindexdict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tooptional.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_unwrap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_way.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_weakmember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_withcontext.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_zip.cpp
+)
diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..96b0c048ea6
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,158 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-comp_nodes-no_llvm)
+target_compile_options(minikql-comp_nodes-no_llvm PRIVATE
+ -mprfchw
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-comp_nodes-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-comp_nodes-no_llvm PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ ydb-library-binary_json
+ library-yql-minikql
+ yql-minikql-arrow
+ public-udf-arrow
+ parser-pg_wrapper-interface
+ library-yql-utils
+ cpp-actors-core
+ minikql-invoke_builtins-no_llvm
+)
+target_sources(minikql-comp_nodes-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_addmember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_aggrcount.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_append.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_count.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_minmax.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_some.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_sum.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_coalesce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_extend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_if.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_just.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_compress.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_func.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_skiptake.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_top.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_blocks.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain1_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_check_args.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_coalesce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_collect.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_contains.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_div.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mod.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mul.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_discard.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_element.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ensure.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_enumerate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_exists.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_extend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flow.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold1.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_frombytes.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromyson.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_guess.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hasitems.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_heap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_if.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_invoke.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterable.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lazy_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_length.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_llvm_base.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lookup.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_mapnext.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_nop.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_now.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_pickle.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_prepend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_random.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reduce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_replicate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reverse.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_rh_hash.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_scalar_apply.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_size.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_source.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_state.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_take.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_timezone.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tobytes.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_toindexdict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tooptional.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_unwrap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_way.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_weakmember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_withcontext.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_zip.cpp
+)
diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.txt b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..f10e5ccc395
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,157 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-comp_nodes-no_llvm)
+target_compile_options(minikql-comp_nodes-no_llvm PRIVATE
+ -mprfchw
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-comp_nodes-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-comp_nodes-no_llvm PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ ydb-library-binary_json
+ library-yql-minikql
+ yql-minikql-arrow
+ public-udf-arrow
+ parser-pg_wrapper-interface
+ library-yql-utils
+ cpp-actors-core
+ minikql-invoke_builtins-no_llvm
+)
+target_sources(minikql-comp_nodes-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_addmember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_aggrcount.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_append.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_count.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_minmax.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_some.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_agg_sum.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_coalesce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_extend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_if.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_just.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_logical.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_compress.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_func.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_skiptake.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_top.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_block_tuple.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_blocks.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_callable.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chain1_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_check_args.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_chopper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_coalesce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_collect.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_contains.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_div.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mod.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_decimal_mul.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_dictitems.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_discard.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_element.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ensure.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_enumerate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_exists.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_extend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_flow.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fold1.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_frombytes.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromstring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_fromyson.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_guess.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hasitems.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_heap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_hopping.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_if.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_ifpresent.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_invoke.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterable.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lazy_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_length.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_listfromrange.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_llvm_base.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_logical.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_lookup.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_mapnext.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multihopping.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_multimap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_next_value.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_nop.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_now.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_null.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_pickle.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_prepend.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_queue.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_random.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_range.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reduce.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_removemember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_replicate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_reverse.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_rh_hash.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_round.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_safe_circular_buffer.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_scalar_apply.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_seq.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_size.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_skip.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_sort.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_source.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_state.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_squeeze_to_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_condense1.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_switch.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_take.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_timezone.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tobytes.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_toindexdict.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tooptional.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_tostring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_unwrap.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_way.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_weakmember.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_while.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chain_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_chopper.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_combine.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_condense.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_filter.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_map.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_wide_top_sort.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_withcontext.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/mkql_zip.cpp
+)
diff --git a/ydb/library/yql/minikql/comp_nodes/no_llvm/ya.make b/ydb/library/yql/minikql/comp_nodes/no_llvm/ya.make
new file mode 100644
index 00000000000..589a87f3541
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/no_llvm/ya.make
@@ -0,0 +1,11 @@
+LIBRARY()
+
+CXXFLAGS(-DMKQL_DISABLE_CODEGEN)
+
+ADDINCL(GLOBAL ydb/library/yql/minikql/codegen/llvm_stub)
+
+INCLUDE(../ya.make.inc)
+
+PEERDIR(ydb/library/yql/minikql/invoke_builtins/no_llvm)
+
+END()
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..4cc7327d09f
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,124 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-comp_nodes-ut)
+target_compile_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ -mprfchw
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-comp_nodes-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ minikql-comp_nodes-llvm
+ yql-public-udf
+ public-udf-arrow
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_test_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-comp_nodes-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-comp_nodes-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-comp_nodes-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-comp_nodes-ut)
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..62b305769ae
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,127 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-comp_nodes-ut)
+target_compile_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ -mprfchw
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-comp_nodes-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ minikql-comp_nodes-llvm
+ yql-public-udf
+ public-udf-arrow
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_test_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-comp_nodes-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-comp_nodes-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-comp_nodes-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-minikql-comp_nodes-ut)
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..795dc42da2a
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,129 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-comp_nodes-ut)
+target_compile_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ -mprfchw
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-comp_nodes-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ minikql-comp_nodes-llvm
+ yql-public-udf
+ public-udf-arrow
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_test_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-comp_nodes-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-comp_nodes-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-comp_nodes-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-minikql-comp_nodes-ut)
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.txt b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..6a530932fed
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,117 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-comp_nodes-ut)
+target_compile_options(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ -mprfchw
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-comp_nodes-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ minikql-comp_nodes-llvm
+ yql-public-udf
+ public-udf-arrow
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_sources(ydb-library-yql-minikql-comp_nodes-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_test_factory.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 60
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-comp_nodes-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-comp_nodes-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-comp_nodes-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-comp_nodes-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-comp_nodes-ut)
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/build_no_codegen.sh b/ydb/library/yql/minikql/comp_nodes/ut/build_no_codegen.sh
new file mode 100755
index 00000000000..9ad6ea4eabe
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/build_no_codegen.sh
@@ -0,0 +1,4 @@
+#!/usr/bin/env bash
+set -ex
+
+ya make -D MKQL_DISABLE_CODEGEN --target-platform=DEFAULT-LINUX-X86_64 --target-platform-flag=CFLAGS='-DMKQL_DISABLE_CODEGEN'
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp
new file mode 100644
index 00000000000..619de191ea9
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_bit_utils_ut.cpp
@@ -0,0 +1,67 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/arrow/mkql_bit_utils.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+ui8 NaiveCompressByte(ui8 value, ui8 mask) {
+ ui8 result = 0;
+ ui8 outPos = 0;
+ for (ui8 i = 0; i < 8; ++i) {
+ if (mask & (1 << i)) {
+ ui8 bit = (value & (1 << i)) != 0;
+ result |= (bit << outPos);
+ ++outPos;
+ }
+ }
+ return result;
+}
+
+} // namespace
+
+
+Y_UNIT_TEST_SUITE(TMiniKQLBitUtilsTest) {
+Y_UNIT_TEST(TestCompressByte) {
+ for (size_t value = 0; value < 256; ++value) {
+ for (size_t mask = 0; mask < 256; ++mask) {
+ UNIT_ASSERT_EQUAL(NaiveCompressByte(value, mask), CompressByte(value, mask));
+ }
+ }
+}
+
+Y_UNIT_TEST(TestLoad) {
+ const ui8 src[] = {0b01110100, 0b11011101, 0b01101011};
+ UNIT_ASSERT_EQUAL(LoadByteUnaligned(src, 10), 0b11110111);
+ UNIT_ASSERT_EQUAL(LoadByteUnaligned(src, 16), 0b01101011);
+}
+
+Y_UNIT_TEST(CompressAligned) {
+ const ui8 data[] = {0b01110100, 0b11011101, 0b01101011};
+ const ui8 mask[] = {0b11101100, 0b10111010, 0b10001111};
+ ui8 result[100];
+ auto res = CompressBitmap(data, 0, mask, 0, result, 0, 24);
+ UNIT_ASSERT_EQUAL(res, 15);
+ UNIT_ASSERT_EQUAL(result[0], 0b11001101);
+ UNIT_ASSERT_EQUAL(result[1] & 0x7fu, 0b00101110);
+}
+
+Y_UNIT_TEST(CompressUnalignedOutput) {
+ const ui8 data[] = {0b01110100, 0b11011101, 0b01101011};
+ const ui8 mask[] = {0b11101100, 0b10111010, 0b10001111};
+ ui8 result[100];
+ result[0] = 0b101;
+ auto res = CompressBitmap(data, 0, mask, 0, result, 3, 24);
+ UNIT_ASSERT_EQUAL(res, 18);
+ UNIT_ASSERT_EQUAL(result[0], 0b01101101);
+ UNIT_ASSERT_EQUAL(result[1], 0b01110110);
+ UNIT_ASSERT_EQUAL(result[2] & 0x3, 0b01);
+}
+
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
+
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp
new file mode 100644
index 00000000000..f68b44d7705
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_compress_ut.cpp
@@ -0,0 +1,227 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+#include <ydb/library/yql/minikql/computation/mkql_block_builder.h>
+
+#include <util/random/random.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+void DoNestedTuplesCompressTest(bool useRandom, bool doFilter) {
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ const auto utf8Type = pb.NewDataType(NUdf::EDataSlot::Utf8);
+
+ const auto innerTupleType = pb.NewTupleType({ui64Type, boolType, utf8Type});
+ const auto outerTupleType = pb.NewTupleType({ui64Type, innerTupleType, utf8Type});
+ const auto finalTupleType = pb.NewTupleType({ui64Type, outerTupleType, boolType});
+
+ const auto resultTupleType = pb.NewTupleType({ui64Type, outerTupleType});
+
+ TVector<TRuntimeNode> items;
+ static_assert(MaxBlockSizeInBytes % 4 == 0);
+ constexpr size_t fixedStrSize = MaxBlockSizeInBytes / 4;
+
+ if (useRandom) {
+ SetRandomSeed(0);
+ }
+
+ for (size_t i = 0; i < 95; ++i) {
+ std::string str;
+ bool filterValue;
+ if (useRandom) {
+ size_t len = RandomNumber<size_t>(2 * MaxBlockSizeInBytes);
+ str.reserve(len);
+ for (size_t i = 0; i < len; ++i) {
+ str.push_back((char)RandomNumber<ui8>(128));
+ }
+ if (doFilter) {
+ filterValue = RandomNumber<ui8>() & 1;
+ } else {
+ filterValue = true;
+ }
+ } else {
+ str = std::string(fixedStrSize, ' ' + i);
+ if (doFilter) {
+ filterValue = (i % 4) < 2;
+ } else {
+ filterValue = true;
+ }
+ }
+
+ auto innerTuple = pb.NewTuple(innerTupleType, {
+ pb.NewDataLiteral<ui64>(i),
+ pb.NewDataLiteral<bool>(i % 2),
+ pb.NewDataLiteral<NUdf::EDataSlot::Utf8>((i % 2) ? str : std::string()),
+ });
+ auto outerTuple = pb.NewTuple(outerTupleType, {
+ pb.NewDataLiteral<ui64>(i),
+ innerTuple,
+ pb.NewDataLiteral<NUdf::EDataSlot::Utf8>((i % 2) ? std::string() : str),
+ });
+
+ auto finalTuple = pb.NewTuple(finalTupleType, {
+ pb.NewDataLiteral<ui64>(i),
+ outerTuple,
+ pb.NewDataLiteral<bool>(filterValue),
+ });
+ items.push_back(finalTuple);
+ }
+
+ const auto list = pb.NewList(finalTupleType, std::move(items));
+
+ auto node = pb.ToFlow(list);
+ node = pb.ExpandMap(node, [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)};
+ });
+ node = pb.WideToBlocks(node);
+
+ node = pb.BlockExpandChunked(node);
+ node = pb.WideSkipBlocks(node, pb.NewDataLiteral<ui64>(19));
+ node = pb.BlockCompress(node, 2);
+ node = pb.WideFromBlocks(node);
+
+ node = pb.NarrowMap(node, [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return pb.NewTuple(resultTupleType, {items[0], items[1]});
+ });
+
+ const auto pgmReturn = pb.ForwardList(node);
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ if (useRandom) {
+ SetRandomSeed(0);
+ }
+
+ for (size_t i = 0; i < 95; ++i) {
+ std::string str;
+ bool filterValue;
+ if (useRandom) {
+ size_t len = RandomNumber<size_t>(2 * MaxBlockSizeInBytes);
+ str.reserve(len);
+ for (size_t i = 0; i < len; ++i) {
+ str.push_back((char)RandomNumber<ui8>(128));
+ }
+ if (doFilter) {
+ filterValue = RandomNumber<ui8>() & 1;
+ } else {
+ filterValue = true;
+ }
+ } else {
+ str = std::string(fixedStrSize, ' ' + i);
+ if (doFilter) {
+ filterValue = (i % 4) < 2;
+ } else {
+ filterValue = true;
+ }
+ }
+
+ if (i < 19 || !filterValue) {
+ continue;
+ }
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ ui64 topNum = item.GetElement(0).Get<ui64>();
+ const auto& outer = item.GetElement(1);
+
+ ui64 num = outer.GetElement(0).Get<ui64>();
+ const auto& inner = outer.GetElement(1);
+
+ auto outerStrVal = outer.GetElement(2);
+ std::string_view outerStr = outerStrVal.AsStringRef();
+
+ ui64 innerNum = inner.GetElement(0).Get<ui64>();
+ bool innerBool = inner.GetElement(1).Get<bool>();
+ auto innerStrVal = inner.GetElement(2);
+
+ std::string_view innerStr = innerStrVal.AsStringRef();
+
+ UNIT_ASSERT_VALUES_EQUAL(num, i);
+ UNIT_ASSERT_VALUES_EQUAL(topNum, i);
+ UNIT_ASSERT_VALUES_EQUAL(innerNum, i);
+ UNIT_ASSERT_VALUES_EQUAL(innerBool, i % 2);
+
+ std::string expectedInner = (i % 2) ? str : std::string();
+ std::string expectedOuter = (i % 2) ? std::string() : str;
+
+ UNIT_ASSERT(innerStr == expectedInner);
+ UNIT_ASSERT(outerStr == expectedOuter);
+ }
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+}
+
+} //namespace
+
+Y_UNIT_TEST_SUITE(TMiniKQLBlockCompressTest) {
+Y_UNIT_TEST(CompressBasic) {
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ const auto tupleType = pb.NewTupleType({boolType, ui64Type, boolType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(false), pb.NewDataLiteral<ui64>(1), pb.NewDataLiteral<bool>(true)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(true), pb.NewDataLiteral<ui64>(2), pb.NewDataLiteral<bool>(false)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(false), pb.NewDataLiteral<ui64>(3), pb.NewDataLiteral<bool>(true)});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(false), pb.NewDataLiteral<ui64>(4), pb.NewDataLiteral<bool>(true)});
+ const auto data5 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(true), pb.NewDataLiteral<ui64>(5), pb.NewDataLiteral<bool>(false)});
+ const auto data6 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(true), pb.NewDataLiteral<ui64>(6), pb.NewDataLiteral<bool>(true)});
+ const auto data7 = pb.NewTuple(tupleType, {pb.NewDataLiteral<bool>(false), pb.NewDataLiteral<ui64>(7), pb.NewDataLiteral<bool>(true)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7});
+ const auto flow = pb.ToFlow(list);
+
+ const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)};
+ });
+ const auto compressedFlow = pb.WideFromBlocks(pb.BlockCompress(pb.WideToBlocks(wideFlow), 0));
+ const auto narrowFlow = pb.NarrowMap(compressedFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return pb.NewTuple({items[0], items[1]});
+ });
+
+ const auto pgmReturn = pb.ForwardList(narrowFlow);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<ui64>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<bool>(), false);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<ui64>(), 5);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<bool>(), false);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<ui64>(), 6);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<bool>(), true);
+
+ UNIT_ASSERT(!iterator.Next(item));
+}
+
+Y_UNIT_TEST(CompressNestedTuples) {
+ DoNestedTuplesCompressTest(false, true);
+ DoNestedTuplesCompressTest(false, false);
+}
+
+Y_UNIT_TEST(CompressNestedTuplesWithRandom) {
+ DoNestedTuplesCompressTest(true, true);
+ DoNestedTuplesCompressTest(true, false);
+}
+
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp
new file mode 100644
index 00000000000..830ee7152ad
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_block_skiptake_ut.cpp
@@ -0,0 +1,179 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/arrow/arrow_defs.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+#include <arrow/array/builder_primitive.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+class TTestBlockFlowWrapper: public TStatefulWideFlowComputationNode<TTestBlockFlowWrapper> {
+ typedef TStatefulWideFlowComputationNode<TTestBlockFlowWrapper> TBaseComputation;
+
+public:
+ TTestBlockFlowWrapper(TComputationMutables& mutables, size_t blockSize, size_t blockCount)
+ : TBaseComputation(mutables, nullptr, EValueRepresentation::Any)
+ , BlockSize(blockSize)
+ , BlockCount(blockCount)
+ {
+ }
+
+ EFetchResult DoCalculate(NUdf::TUnboxedValue& state, TComputationContext& ctx, NUdf::TUnboxedValue*const* output) const {
+ if (!state.HasValue()) {
+ state = NUdf::TUnboxedValue::Zero();
+ }
+
+ ui64 index = state.Get<ui64>();
+ if (index >= BlockCount) {
+ return EFetchResult::Finish;
+ }
+
+ arrow::UInt64Builder builder(&ctx.ArrowMemoryPool);
+ ARROW_OK(builder.Reserve(BlockSize));
+ for (size_t i = 0; i < BlockSize; ++i) {
+ builder.UnsafeAppend(index * BlockSize + i);
+ }
+
+ std::shared_ptr<arrow::ArrayData> block;
+ ARROW_OK(builder.FinishInternal(&block));
+
+ *output[0] = ctx.HolderFactory.CreateArrowBlock(std::move(block));
+ *output[1] = ctx.HolderFactory.CreateArrowBlock(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(index)));
+ *output[2] = ctx.HolderFactory.CreateArrowBlock(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(BlockSize)));
+
+ state = NUdf::TUnboxedValuePod(++index);
+ return EFetchResult::One;
+ }
+
+private:
+ void RegisterDependencies() const final {
+ }
+
+ const size_t BlockSize;
+ const size_t BlockCount;
+};
+
+IComputationNode* WrapTestBlockFlow(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ MKQL_ENSURE(callable.GetInputsCount() == 0, "Expected no args");
+ return new TTestBlockFlowWrapper(ctx.Mutables, 5, 2);
+}
+
+TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
+ return CreateDeterministicRandomProvider(1);
+}
+
+TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
+ return CreateDeterministicTimeProvider(10000000);
+}
+
+TComputationNodeFactory GetTestFactory() {
+ return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+ if (callable.GetType()->GetName() == "TestBlockFlow") {
+ return WrapTestBlockFlow(callable, ctx);
+ }
+ return GetBuiltinFactory()(callable, ctx);
+ };
+}
+
+struct TSetup_ {
+ TSetup_()
+ : Alloc(__LOCATION__)
+ {
+ FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ RandomProvider = CreateRandomProvider();
+ TimeProvider = CreateTimeProvider();
+
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
+ TAutoPtr<IComputationGraph> BuildGraph(TRuntimeNode pgm, EGraphPerProcess graphPerProcess = EGraphPerProcess::Multi, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
+ Explorer.Walk(pgm.GetNode(), *Env);
+ TComputationPatternOpts opts(Alloc.Ref(), *Env, GetTestFactory(), FunctionRegistry.Get(),
+ NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, "OFF", graphPerProcess);
+ Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
+ return Pattern->Clone(opts.ToComputationOptions(*RandomProvider, *TimeProvider));
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+
+ TScopedAlloc Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
+ IComputationPattern::TPtr Pattern;
+};
+
+TRuntimeNode MakeFlow(TSetup_& setup) {
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ TCallableBuilder callableBuilder(*setup.Env, "TestBlockFlow",
+ pb.NewFlowType(
+ pb.NewMultiType({
+ pb.NewBlockType(pb.NewDataType(NUdf::EDataSlot::Uint64), TBlockType::EShape::Many),
+ pb.NewBlockType(pb.NewDataType(NUdf::EDataSlot::Uint64), TBlockType::EShape::Scalar),
+ pb.NewBlockType(pb.NewDataType(NUdf::EDataSlot::Uint64), TBlockType::EShape::Scalar),
+ })));
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
+} // namespace
+
+
+Y_UNIT_TEST_SUITE(TMiniKQLWideTakeSkipBlocks) {
+ Y_UNIT_TEST(TestWideTakeSkipBlocks) {
+ TSetup_ setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto flow = MakeFlow(setup);
+
+ const auto part = pb.WideTakeBlocks(pb.WideSkipBlocks(flow, pb.NewDataLiteral<ui64>(3)), pb.NewDataLiteral<ui64>(5));
+ const auto plain = pb.WideFromBlocks(part);
+
+ const auto singleValueFlow = pb.NarrowMap(plain, [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ // 0, 0;
+ // 1, 0;
+ // 2, 0;
+ // 3, 0; -> 3
+ // 4, 0; -> 4
+ // 5, 1; -> 6
+ // 6, 1; -> 7
+ // 7, 1; -> 8
+ // 8, 1;
+ // 9, 1;
+ // 10, 1;
+ return pb.Add(items[0], items[1]);
+ });
+
+ const auto pgmReturn = pb.ForwardList(singleValueFlow);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 3);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 4);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 6);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 7);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 8);
+ }
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
+
+
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp
new file mode 100644
index 00000000000..da7a1dceed8
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_blocks_ut.cpp
@@ -0,0 +1,737 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+#include <arrow/compute/exec_internal.h>
+#include <arrow/array/builder_primitive.h>
+
+#include <ydb/library/yql/public/udf/udf_helpers.h>
+#include <ydb/library/yql/public/udf/arrow/udf_arrow_helpers.h>
+
+BEGIN_SIMPLE_ARROW_UDF(TInc, i32(i32)) {
+ Y_UNUSED(valueBuilder);
+ const i32 arg = args[0].Get<i32>();
+ return NYql::NUdf::TUnboxedValuePod(arg + 1);
+}
+
+struct TIncKernelExec : public NYql::NUdf::TUnaryKernelExec<TIncKernelExec> {
+ template <typename TSink>
+ static void Process(NYql::NUdf::TBlockItem arg, const TSink& sink) {
+ sink(NYql::NUdf::TBlockItem(arg.As<i32>() + 1));
+ }
+};
+
+END_SIMPLE_ARROW_UDF(TInc, TIncKernelExec::Do);
+
+SIMPLE_MODULE(TBlockUTModule,
+ TInc
+)
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+ arrow::Datum ExecuteOneKernel(const IArrowKernelComputationNode* kernelNode,
+ const std::vector<arrow::Datum>& argDatums, arrow::compute::ExecContext& execContext) {
+ const auto& kernel = kernelNode->GetArrowKernel();
+ arrow::compute::KernelContext kernelContext(&execContext);
+ std::unique_ptr<arrow::compute::KernelState> state;
+ if (kernel.init) {
+ state = ARROW_RESULT(kernel.init(&kernelContext, { &kernel, kernelNode->GetArgsDesc(), nullptr }));
+ kernelContext.SetState(state.get());
+ }
+
+ auto executor = arrow::compute::detail::KernelExecutor::MakeScalar();
+ ARROW_OK(executor->Init(&kernelContext, { &kernel, kernelNode->GetArgsDesc(), nullptr }));
+ auto listener = std::make_shared<arrow::compute::detail::DatumAccumulator>();
+ ARROW_OK(executor->Execute(argDatums, listener.get()));
+ return executor->WrapResults(argDatums, listener->values());
+ }
+
+ void ExecuteAllKernels(std::vector<arrow::Datum>& datums, const TArrowKernelsTopology* topology, arrow::compute::ExecContext& execContext) {
+ for (ui32 i = 0; i < topology->Items.size(); ++i) {
+ std::vector<arrow::Datum> argDatums;
+ argDatums.reserve(topology->Items[i].Inputs.size());
+ for (auto j : topology->Items[i].Inputs) {
+ argDatums.emplace_back(datums[j]);
+ }
+
+ arrow::Datum output = ExecuteOneKernel(topology->Items[i].Node.get(), argDatums, execContext);
+ datums[i + topology->InputArgsCount] = output;
+ }
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLBlocksTest) {
+Y_UNIT_TEST(TestEmpty) {
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto list = pb.NewEmptyList(type);
+ const auto sourceFlow = pb.ToFlow(list);
+ const auto flowAfterBlocks = pb.FromBlocks(pb.ToBlocks(sourceFlow));
+ const auto pgmReturn = pb.ForwardList(flowAfterBlocks);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(!iterator.Next(item));
+}
+
+Y_UNIT_TEST(TestSimple) {
+ static const size_t dataCount = 1000;
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ auto data = TVector<TRuntimeNode>(Reserve(dataCount));
+ for (size_t i = 0; i < dataCount; ++i) {
+ data.push_back(pb.NewDataLiteral<ui64>(i));
+ }
+ const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto list = pb.NewList(type, data);
+ const auto sourceFlow = pb.ToFlow(list);
+ const auto flowAfterBlocks = pb.FromBlocks(pb.ToBlocks(sourceFlow));
+ const auto pgmReturn = pb.ForwardList(flowAfterBlocks);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ for (size_t i = 0; i < dataCount; ++i) {
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), i);
+ }
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(!iterator.Next(item));
+}
+
+Y_UNIT_TEST(TestWideToBlocks) {
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto tupleType = pb.NewTupleType({ui64Type, ui64Type});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(1), pb.NewDataLiteral<ui64>(10)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(2), pb.NewDataLiteral<ui64>(20)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(3), pb.NewDataLiteral<ui64>(30)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3});
+ const auto flow = pb.ToFlow(list);
+
+ const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ return {pb.Nth(item, 0U), pb.Nth(item, 1U)};
+ });
+ const auto wideBlocksFlow = pb.WideToBlocks(wideFlow);
+ const auto narrowBlocksFlow = pb.NarrowMap(wideBlocksFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return items[1];
+ });
+ const auto narrowFlow = pb.FromBlocks(narrowBlocksFlow);
+ const auto pgmReturn = pb.ForwardList(narrowFlow);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 10);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 20);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 30);
+}
+
+namespace {
+void TestChunked(bool withBlockExpand) {
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ const auto stringType = pb.NewDataType(NUdf::EDataSlot::String);
+ const auto utf8Type = pb.NewDataType(NUdf::EDataSlot::Utf8);
+
+ const auto tupleType = pb.NewTupleType({ui64Type, boolType, stringType, utf8Type});
+
+ TVector<TRuntimeNode> items;
+ const size_t bigStrSize = 1024 * 1024 + 100;
+ const size_t smallStrSize = 256 * 1024;
+ for (size_t i = 0; i < 20; ++i) {
+
+ if (i % 2 == 0) {
+ std::string big(bigStrSize, '0' + i);
+ std::string small(smallStrSize, 'A' + i);
+
+ items.push_back(pb.NewTuple(tupleType, { pb.NewDataLiteral<ui64>(i), pb.NewDataLiteral<bool>(true),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>(big),
+ pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(small),
+ }));
+ } else {
+ items.push_back(pb.NewTuple(tupleType, { pb.NewDataLiteral<ui64>(i), pb.NewDataLiteral<bool>(false),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>(""),
+ pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(""),
+ }));
+
+ }
+ }
+
+ const auto list = pb.NewList(tupleType, std::move(items));
+
+ auto node = pb.ToFlow(list);
+ node = pb.ExpandMap(node, [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U)};
+ });
+ node = pb.WideToBlocks(node);
+ if (withBlockExpand) {
+ node = pb.BlockExpandChunked(node);
+ // WideTakeBlocks won't work on chunked blocks
+ node = pb.WideTakeBlocks(node, pb.NewDataLiteral<ui64>(19));
+ node = pb.WideFromBlocks(node);
+ } else {
+ // WideFromBlocks should support chunked blocks
+ node = pb.WideFromBlocks(node);
+ node = pb.Take(node, pb.NewDataLiteral<ui64>(19));
+ }
+ node = pb.NarrowMap(node, [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return pb.NewTuple(tupleType, {items[0], items[1], items[2], items[3]});
+ });
+
+ const auto pgmReturn = pb.ForwardList(node);
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ for (size_t i = 0; i < 19; ++i) {
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ ui64 num = item.GetElement(0).Get<ui64>();
+ bool bl = item.GetElement(1).Get<bool>();
+ auto strVal = item.GetElement(2);
+ auto utf8Val = item.GetElement(3);
+ std::string_view str = strVal.AsStringRef();
+ std::string_view utf8 = utf8Val.AsStringRef();
+
+ UNIT_ASSERT_VALUES_EQUAL(num, i);
+ UNIT_ASSERT_VALUES_EQUAL(bl, i % 2 == 0);
+ if (i % 2 == 0) {
+ std::string big(bigStrSize, '0' + i);
+ std::string small(smallStrSize, 'A' + i);
+ UNIT_ASSERT_VALUES_EQUAL(str, big);
+ UNIT_ASSERT_VALUES_EQUAL(utf8, small);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(str.size(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(utf8.size(), 0);
+ }
+ }
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+
+}
+
+} // namespace
+
+Y_UNIT_TEST(TestBlockExpandChunked) {
+ TestChunked(true);
+}
+
+Y_UNIT_TEST(TestWideFromBlocksForChunked) {
+ TestChunked(false);
+}
+
+Y_UNIT_TEST(TestScalar) {
+ const ui64 testValue = 42;
+
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ auto dataLiteral = pb.NewDataLiteral<ui64>(testValue);
+ const auto dataAfterBlocks = pb.AsScalar(dataLiteral);
+
+ const auto graph = setup.BuildGraph(dataAfterBlocks);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT(value.HasValue() && value.IsBoxed());
+ UNIT_ASSERT_VALUES_EQUAL(TArrowBlock::From(value).GetDatum().scalar_as<arrow::UInt64Scalar>().value, testValue);
+}
+
+Y_UNIT_TEST(TestBlockFunc) {
+ TSetup<false> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto tupleType = pb.NewTupleType({ui64Type, ui64Type});
+ const auto ui64BlockType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many);
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(1), pb.NewDataLiteral<ui64>(10)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(2), pb.NewDataLiteral<ui64>(20)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(3), pb.NewDataLiteral<ui64>(30)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3});
+ const auto flow = pb.ToFlow(list);
+
+ const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ return {pb.Nth(item, 0U), pb.Nth(item, 1U)};
+ });
+ const auto wideBlocksFlow = pb.WideToBlocks(wideFlow);
+ const auto sumWideFlow = pb.WideMap(wideBlocksFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {pb.BlockFunc("Add", ui64BlockType, {items[0], items[1]})};
+ });
+ const auto sumNarrowFlow = pb.NarrowMap(sumWideFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return items[0];
+ });
+ const auto pgmReturn = pb.Collect(pb.FromBlocks(sumNarrowFlow));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 11);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 22);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 33);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+}
+
+Y_UNIT_TEST(TestBlockFuncWithNullables) {
+ TSetup<false> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto optionalUi64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id, true);
+ const auto tupleType = pb.NewTupleType({optionalUi64Type, optionalUi64Type});
+ const auto emptyOptionalUi64 = pb.NewEmptyOptional(optionalUi64Type);
+ const auto ui64OptBlockType = pb.NewBlockType(optionalUi64Type, TBlockType::EShape::Many);
+
+ const auto data1 = pb.NewTuple(tupleType, {
+ pb.NewOptional(pb.NewDataLiteral<ui64>(1)),
+ emptyOptionalUi64
+ });
+ const auto data2 = pb.NewTuple(tupleType, {
+ emptyOptionalUi64,
+ pb.NewOptional(pb.NewDataLiteral<ui64>(20))
+ });
+ const auto data3 = pb.NewTuple(tupleType, {
+ emptyOptionalUi64,
+ emptyOptionalUi64
+ });
+ const auto data4 = pb.NewTuple(tupleType, {
+ pb.NewOptional(pb.NewDataLiteral<ui64>(10)),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(20))
+ });
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+ const auto flow = pb.ToFlow(list);
+
+ const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ return {pb.Nth(item, 0U), pb.Nth(item, 1U)};
+ });
+ const auto wideBlocksFlow = pb.WideToBlocks(wideFlow);
+ const auto sumWideFlow = pb.WideMap(wideBlocksFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {pb.BlockFunc("Add", ui64OptBlockType, {items[0], items[1]})};
+ });
+ const auto sumNarrowFlow = pb.NarrowMap(sumWideFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return items[0];
+ });
+ const auto pgmReturn = pb.Collect(pb.FromBlocks(sumNarrowFlow));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 30);
+ UNIT_ASSERT(!iterator.Next(item));
+}
+
+Y_UNIT_TEST(TestBlockFuncWithNullableScalar) {
+ TSetup<false> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto optionalUi64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id, true);
+ const auto ui64OptBlockType = pb.NewBlockType(optionalUi64Type, TBlockType::EShape::Many);
+ const auto emptyOptionalUi64 = pb.NewEmptyOptional(optionalUi64Type);
+
+ const auto list = pb.NewList(optionalUi64Type, {
+ pb.NewOptional(pb.NewDataLiteral<ui64>(10)),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(20)),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(30))
+ });
+ const auto flow = pb.ToFlow(list);
+ const auto blocksFlow = pb.ToBlocks(flow);
+
+ THolder<IComputationGraph> graph;
+ auto map = [&](const TProgramBuilder::TUnaryLambda& func) {
+ const auto pgmReturn = pb.Collect(pb.FromBlocks(pb.Map(blocksFlow, func)));
+ graph = setup.BuildGraph(pgmReturn);
+ return graph->GetValue().GetListIterator();
+ };
+
+ {
+ const auto scalar = pb.AsScalar(emptyOptionalUi64);
+ auto iterator = map([&](TRuntimeNode item) -> TRuntimeNode {
+ return {pb.BlockFunc("Add", ui64OptBlockType, {scalar, item})};
+ });
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ {
+ const auto scalar = pb.AsScalar(emptyOptionalUi64);
+ auto iterator = map([&](TRuntimeNode item) -> TRuntimeNode {
+ return {pb.BlockFunc("Add", ui64OptBlockType, {item, scalar})};
+ });
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ {
+ const auto scalar = pb.AsScalar(pb.NewDataLiteral<ui64>(100));
+ auto iterator = map([&](TRuntimeNode item) -> TRuntimeNode {
+ return {pb.BlockFunc("Add", ui64OptBlockType, {item, scalar})};
+ });
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 110);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 120);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 130);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+
+Y_UNIT_TEST(TestBlockFuncWithScalar) {
+ TSetup<false> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto ui64BlockType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many);
+
+ const auto data1 = pb.NewDataLiteral<ui64>(10);
+ const auto data2 = pb.NewDataLiteral<ui64>(20);
+ const auto data3 = pb.NewDataLiteral<ui64>(30);
+ const auto rightScalar = pb.AsScalar(pb.NewDataLiteral<ui64>(100));
+ const auto leftScalar = pb.AsScalar(pb.NewDataLiteral<ui64>(1000));
+
+ const auto list = pb.NewList(ui64Type, {data1, data2, data3});
+ const auto flow = pb.ToFlow(list);
+ const auto blocksFlow = pb.ToBlocks(flow);
+ const auto sumBlocksFlow = pb.Map(blocksFlow, [&](TRuntimeNode item) -> TRuntimeNode {
+ return {pb.BlockFunc("Add", ui64BlockType, { leftScalar, {pb.BlockFunc("Add", ui64BlockType, { item, rightScalar } )}})};
+ });
+ const auto pgmReturn = pb.Collect(pb.FromBlocks(sumBlocksFlow));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 1110);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 1120);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 1130);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+}
+
+Y_UNIT_TEST(TestWideFromBlocks) {
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto tupleType = pb.NewTupleType({ui64Type, ui64Type});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(1), pb.NewDataLiteral<ui64>(10)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(2), pb.NewDataLiteral<ui64>(20)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(3), pb.NewDataLiteral<ui64>(30)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3});
+ const auto flow = pb.ToFlow(list);
+
+ const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ return {pb.Nth(item, 0U), pb.Nth(item, 1U)};
+ });
+ const auto wideBlocksFlow = pb.WideToBlocks(wideFlow);
+ const auto wideFlow2 = pb.WideFromBlocks(wideBlocksFlow);
+ const auto narrowFlow = pb.NarrowMap(wideFlow2, [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return items[1];
+ });
+
+ const auto pgmReturn = pb.ForwardList(narrowFlow);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 10);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 20);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 30);
+}
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLDirectKernelTest) {
+Y_UNIT_TEST(Simple) {
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto boolBlocksType = pb.NewBlockType(boolType, TBlockType::EShape::Many);
+ const auto ui64BlocksType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many);
+ const auto arg1 = pb.Arg(boolBlocksType);
+ const auto arg2 = pb.Arg(ui64BlocksType);
+ const auto arg3 = pb.Arg(ui64BlocksType);
+ const auto ifNode = pb.BlockIf(arg1, arg2, arg3);
+ const auto eqNode = pb.BlockFunc("Equals", boolBlocksType, { ifNode, arg2 });
+
+ const auto graph = setup.BuildGraph(eqNode, {arg1.GetNode(), arg2.GetNode(), arg3.GetNode()});
+ const auto topology = graph->GetKernelsTopology();
+ UNIT_ASSERT(topology);
+ UNIT_ASSERT_VALUES_EQUAL(topology->InputArgsCount, 3);
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items.size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Node->GetKernelName(), "If");
+ const std::vector<ui32> expectedInputs1{{0, 1, 2}};
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Inputs, expectedInputs1);
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[1].Node->GetKernelName(), "Equals");
+ const std::vector<ui32> expectedInputs2{{3, 1}};
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[1].Inputs, expectedInputs2);
+
+ arrow::compute::ExecContext execContext;
+ const size_t blockSize = 100000;
+ std::vector<arrow::Datum> datums(topology->InputArgsCount + topology->Items.size());
+ {
+ arrow::UInt8Builder builder1(execContext.memory_pool());
+ arrow::UInt64Builder builder2(execContext.memory_pool()), builder3(execContext.memory_pool());
+ ARROW_OK(builder1.Reserve(blockSize));
+ ARROW_OK(builder2.Reserve(blockSize));
+ ARROW_OK(builder3.Reserve(blockSize));
+ for (size_t i = 0; i < blockSize; ++i) {
+ builder1.UnsafeAppend(i & 1);
+ builder2.UnsafeAppend(i);
+ builder3.UnsafeAppend(3 * i);
+ }
+
+ std::shared_ptr<arrow::ArrayData> data1;
+ ARROW_OK(builder1.FinishInternal(&data1));
+ std::shared_ptr<arrow::ArrayData> data2;
+ ARROW_OK(builder2.FinishInternal(&data2));
+ std::shared_ptr<arrow::ArrayData> data3;
+ ARROW_OK(builder3.FinishInternal(&data3));
+ datums[0] = data1;
+ datums[1] = data2;
+ datums[2] = data3;
+ }
+
+ ExecuteAllKernels(datums, topology, execContext);
+
+ auto res = datums.back().array()->GetValues<ui8>(1);
+ for (size_t i = 0; i < blockSize; ++i) {
+ auto expected = (((i & 1) ? i : i * 3) == i) ? 1 : 0;
+ UNIT_ASSERT_VALUES_EQUAL(res[i], expected);
+ }
+}
+
+Y_UNIT_TEST(WithScalars) {
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto ui64BlocksType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many);
+ const auto scalar = pb.AsScalar(pb.NewDataLiteral(false));
+ const auto arg1 = pb.Arg(ui64BlocksType);
+ const auto arg2 = pb.Arg(ui64BlocksType);
+ const auto ifNode = pb.BlockIf(scalar, arg1, arg2);
+
+ const auto graph = setup.BuildGraph(ifNode, {arg1.GetNode(), arg2.GetNode()});
+ const auto topology = graph->GetKernelsTopology();
+ UNIT_ASSERT(topology);
+ UNIT_ASSERT_VALUES_EQUAL(topology->InputArgsCount, 2);
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items.size(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Node->GetKernelName(), "AsScalar");
+ const std::vector<ui32> expectedInputs1;
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Inputs, expectedInputs1);
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[1].Node->GetKernelName(), "If");
+ const std::vector<ui32> expectedInputs2{{2, 0, 1}};
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[1].Inputs, expectedInputs2);
+
+ arrow::compute::ExecContext execContext;
+ const size_t blockSize = 100000;
+ std::vector<arrow::Datum> datums(topology->InputArgsCount + topology->Items.size());
+ {
+ arrow::UInt64Builder builder1(execContext.memory_pool()), builder2(execContext.memory_pool());
+ ARROW_OK(builder1.Reserve(blockSize));
+ ARROW_OK(builder2.Reserve(blockSize));
+ for (size_t i = 0; i < blockSize; ++i) {
+ builder1.UnsafeAppend(i);
+ builder2.UnsafeAppend(3 * i);
+ }
+
+ std::shared_ptr<arrow::ArrayData> data1;
+ ARROW_OK(builder1.FinishInternal(&data1));
+ std::shared_ptr<arrow::ArrayData> data2;
+ ARROW_OK(builder2.FinishInternal(&data2));
+ datums[0] = data1;
+ datums[1] = data2;
+ }
+
+ ExecuteAllKernels(datums, topology, execContext);
+
+ auto res = datums.back().array()->GetValues<ui64>(1);
+ for (size_t i = 0; i < blockSize; ++i) {
+ auto expected = 3 * i;
+ UNIT_ASSERT_VALUES_EQUAL(res[i], expected);
+ }
+}
+
+Y_UNIT_TEST(Udf) {
+ TVector<TUdfModuleInfo> modules;
+ modules.emplace_back(TUdfModuleInfo{"", "BlockUT", new TBlockUTModule()});
+ TSetup<false> setup({}, std::move(modules));
+
+ auto& pb = *setup.PgmBuilder;
+
+ const auto i32Type = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto i32BlocksType = pb.NewBlockType(i32Type, TBlockType::EShape::Many);
+ const auto arg1 = pb.Arg(i32BlocksType);
+ const auto userType = pb.NewTupleType({
+ pb.NewTupleType({i32BlocksType}),
+ pb.NewEmptyStructType(),
+ pb.NewEmptyTupleType()});
+ const auto udf = pb.Udf("BlockUT.Inc_BlocksImpl", pb.NewVoid(), userType);
+ const auto apply = pb.Apply(udf, {arg1});
+
+ const auto graph = setup.BuildGraph(apply, {arg1.GetNode() });
+ const auto topology = graph->GetKernelsTopology();
+ UNIT_ASSERT(topology);
+ UNIT_ASSERT_VALUES_EQUAL(topology->InputArgsCount, 1);
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items.size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Node->GetKernelName(), "Apply");
+ const std::vector<ui32> expectedInputs1{0};
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Inputs, expectedInputs1);
+
+ arrow::compute::ExecContext execContext;
+ const size_t blockSize = 100000;
+ std::vector<arrow::Datum> datums(topology->InputArgsCount + topology->Items.size());
+ {
+ arrow::Int32Builder builder1(execContext.memory_pool());
+ ARROW_OK(builder1.Reserve(blockSize));
+ for (size_t i = 0; i < blockSize; ++i) {
+ builder1.UnsafeAppend(i);
+ }
+
+ std::shared_ptr<arrow::ArrayData> data1;
+ ARROW_OK(builder1.FinishInternal(&data1));
+ datums[0] = data1;
+ }
+
+ ExecuteAllKernels(datums, topology, execContext);
+
+ auto res = datums.back().array()->GetValues<i32>(1);
+ for (size_t i = 0; i < blockSize; ++i) {
+ auto expected = i + 1;
+ UNIT_ASSERT_VALUES_EQUAL(res[i], expected);
+ }
+}
+
+Y_UNIT_TEST(ScalarApply) {
+ TSetup<false> setup;
+ auto& pb = *setup.PgmBuilder;
+
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto ui64BlocksType = pb.NewBlockType(ui64Type, TBlockType::EShape::Many);
+ const auto arg1 = pb.Arg(ui64BlocksType);
+ const auto arg2 = pb.Arg(ui64BlocksType);
+ const auto scalarApply = pb.ScalarApply({arg1,arg2}, [&](auto args){
+ return pb.Add(args[0], args[1]);
+ });
+
+ const auto graph = setup.BuildGraph(scalarApply, {arg1.GetNode(), arg2.GetNode()});
+ const auto topology = graph->GetKernelsTopology();
+ UNIT_ASSERT(topology);
+ UNIT_ASSERT_VALUES_EQUAL(topology->InputArgsCount, 2);
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items.size(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Node->GetKernelName(), "ScalarApply");
+ const std::vector<ui32> expectedInputs1{{0, 1}};
+ UNIT_ASSERT_VALUES_EQUAL(topology->Items[0].Inputs, expectedInputs1);
+
+ arrow::compute::ExecContext execContext;
+ const size_t blockSize = 100000;
+ std::vector<arrow::Datum> datums(topology->InputArgsCount + topology->Items.size());
+ {
+ arrow::UInt64Builder builder1(execContext.memory_pool()), builder2(execContext.memory_pool());
+ ARROW_OK(builder1.Reserve(blockSize));
+ ARROW_OK(builder2.Reserve(blockSize));
+ for (size_t i = 0; i < blockSize; ++i) {
+ builder1.UnsafeAppend(i);
+ builder2.UnsafeAppend(2 * i);
+ }
+
+ std::shared_ptr<arrow::ArrayData> data1;
+ ARROW_OK(builder1.FinishInternal(&data1));
+ std::shared_ptr<arrow::ArrayData> data2;
+ ARROW_OK(builder2.FinishInternal(&data2));
+ datums[0] = data1;
+ datums[1] = data2;
+ }
+
+ ExecuteAllKernels(datums, topology, execContext);
+
+ auto res = datums.back().array()->GetValues<ui64>(1);
+ for (size_t i = 0; i < blockSize; ++i) {
+ auto expected = 3 * i;
+ UNIT_ASSERT_VALUES_EQUAL(res[i], expected);
+ }
+}
+
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp
new file mode 100644
index 00000000000..b41a0527de8
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_chain_map_ut.cpp
@@ -0,0 +1,254 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLChainMapNodeTest) {
+ Y_UNIT_TEST_LLVM(TestOverList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto dataType = pb.NewOptionalType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)}));
+
+ auto data0 = pb.NewEmptyOptional(dataType);
+
+ auto data2 = pb.NewOptional(pb.NewTuple({
+ pb.NewDataLiteral<i32>(7),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("A")
+ }));
+ auto data3 = pb.NewOptional(pb.NewTuple({
+ pb.NewDataLiteral<i32>(1),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("D")
+ }));
+
+ auto list = pb.NewList(dataType, {data2, data0, data3});
+
+ auto init = pb.NewTuple({
+ pb.NewOptional(pb.NewDataLiteral<i32>(3)),
+ pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("B"))
+ });
+
+ auto pgmReturn = pb.ChainMap(list, init,
+ [&](TRuntimeNode item, TRuntimeNode state) -> TRuntimeNodePair {
+ auto key = pb.Nth(item, 0);
+ auto val = pb.Nth(item, 1);
+ auto skey = pb.AggrAdd(pb.Nth(state, 0), key);
+ auto sval = pb.AggrConcat(pb.Nth(state, 1), val);
+ return {pb.NewTuple({key, val, skey, sval}), pb.NewTuple({skey, sval})};
+ }
+ );
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 7);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "A");
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D");
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 11);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BAD");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(Test1OverList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto dataType = pb.NewOptionalType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)}));
+
+ auto data0 = pb.NewEmptyOptional(dataType);
+
+ auto data1 = pb.NewOptional(pb.NewTuple({
+ pb.NewDataLiteral<i32>(3),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("B")
+ }));
+ auto data2 = pb.NewOptional(pb.NewTuple({
+ pb.NewDataLiteral<i32>(7),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("A")
+ }));
+ auto data3 = pb.NewOptional(pb.NewTuple({
+ pb.NewDataLiteral<i32>(1),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("D")
+ }));
+
+ auto list = pb.NewList(dataType, {data1, data2, data3, data0});
+
+ auto pgmReturn = pb.Chain1Map(list,
+ [&](TRuntimeNode item) -> TRuntimeNodePair {
+ auto key = pb.Nth(item, 0);
+ auto val = pb.Nth(item, 1);
+ return {pb.NewTuple({key, val, key, val}), pb.NewTuple({key, val})};
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) -> TRuntimeNodePair {
+ auto key = pb.Nth(item, 0);
+ auto val = pb.Nth(item, 1);
+ auto skey = pb.Add(pb.Nth(state, 0), key);
+ auto sval = pb.Concat(pb.Nth(state, 1), val);
+ return {pb.NewTuple({key, val, skey, sval}), pb.NewTuple({skey, sval})};
+ }
+ );
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 3);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "B");
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 3);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "B");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 7);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "A");
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D");
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 11);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BAD");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(!item.GetElement(3));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto dataType = pb.NewOptionalType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)}));
+
+ auto data0 = pb.NewEmptyOptional(dataType);
+
+ auto data2 = pb.NewOptional(pb.NewTuple({
+ pb.NewDataLiteral<i32>(7),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("A")
+ }));
+ auto data3 = pb.NewOptional(pb.NewTuple({
+ pb.NewDataLiteral<i32>(1),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("D")
+ }));
+
+ auto list = pb.NewList(dataType, {data2, data0, data3});
+
+ auto init = pb.NewTuple({
+ pb.NewOptional(pb.NewDataLiteral<i32>(3)),
+ pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("B"))
+ });
+
+ auto pgmReturn = pb.FromFlow(pb.ChainMap(pb.ToFlow(list), init,
+ [&](TRuntimeNode item, TRuntimeNode state) -> TRuntimeNodePair {
+ auto key = pb.Nth(item, 0);
+ auto val = pb.Nth(item, 1);
+ auto skey = pb.AggrAdd(pb.Nth(state, 0), key);
+ auto sval = pb.AggrConcat(pb.Nth(state, 1), val);
+ return {pb.NewTuple({key, val, skey, sval}), pb.NewTuple({skey, sval})};
+ }
+ ));
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 7);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "A");
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D");
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 11);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BAD");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(Test1OverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto dataType = pb.NewOptionalType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)}));
+
+ auto data0 = pb.NewEmptyOptional(dataType);
+
+ auto data1 = pb.NewOptional(pb.NewTuple({
+ pb.NewDataLiteral<i32>(3),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("B")
+ }));
+ auto data2 = pb.NewOptional(pb.NewTuple({
+ pb.NewDataLiteral<i32>(7),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("A")
+ }));
+ auto data3 = pb.NewOptional(pb.NewTuple({
+ pb.NewDataLiteral<i32>(1),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("D")
+ }));
+
+ auto list = pb.NewList(dataType, {data1, data2, data3, data0});
+
+ auto pgmReturn = pb.FromFlow(pb.Chain1Map(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNodePair {
+ auto key = pb.Nth(item, 0);
+ auto val = pb.Nth(item, 1);
+ return {pb.NewTuple({key, val, key, val}), pb.NewTuple({key, val})};
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) -> TRuntimeNodePair {
+ auto key = pb.Nth(item, 0);
+ auto val = pb.Nth(item, 1);
+ auto skey = pb.Add(pb.Nth(state, 0), key);
+ auto sval = pb.Concat(pb.Nth(state, 1), val);
+ return {pb.NewTuple({key, val, skey, sval}), pb.NewTuple({skey, sval})};
+ }
+ ));
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 3);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "B");
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 3);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "B");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 7);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "A");
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 10);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BA");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D");
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 11);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3), "BAD");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(!item.GetElement(3));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp
new file mode 100644
index 00000000000..8d9d5576696
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp
@@ -0,0 +1,487 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+template<bool UseLLVM>
+TRuntimeNode MakeStream(TSetup<UseLLVM>& setup, ui64 count = 9U) {
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TCallableBuilder callableBuilder(*setup.Env, "TestStream",
+ pb.NewStreamType(
+ pb.NewDataType(NUdf::EDataSlot::Uint64)
+ )
+ );
+
+ callableBuilder.Add(pb.NewDataLiteral(count));
+
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
+template<bool UseLLVM>
+TRuntimeNode MakeFlow(TSetup<UseLLVM>& setup, ui64 count = 9U) {
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ return pb.ToFlow(MakeStream<UseLLVM>(setup, count));
+}
+
+template<bool UseLLVM>
+TRuntimeNode GroupWithBomb(TSetup<UseLLVM>& setup, TRuntimeNode stream) {
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto keyExtractor = [&](TRuntimeNode item) { return item; };
+ const auto groupSwitch = [&](TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); };
+
+ return pb.Chopper(stream, keyExtractor, groupSwitch, [&](TRuntimeNode, TRuntimeNode group) {
+ const auto bomb = pb.NewDataLiteral<NUdf::EDataSlot::String>("BOMB");
+ return pb.Ensure(pb.Map(group, [&] (TRuntimeNode) { return bomb; }), pb.NewDataLiteral<bool>(false), bomb, "", 0, 0);
+ });
+}
+
+template<bool UseLLVM>
+TRuntimeNode Group(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) {
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto keyExtractor = [&](TRuntimeNode item) { return item; };
+
+ return pb.Chopper(stream, keyExtractor, groupSwitch, [&](TRuntimeNode key, TRuntimeNode grpItem) {
+ return pb.Condense(grpItem, pb.NewDataLiteral<NUdf::EDataSlot::String>("*"),
+ [&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); },
+ [&] (TRuntimeNode item, TRuntimeNode state) {
+ auto res = pb.Concat(pb.ToString(key), pb.ToString(item));
+ res = pb.Concat(state, res);
+ return pb.Concat(res, pb.NewDataLiteral<NUdf::EDataSlot::String>("*"));
+ });
+ });
+}
+
+template<bool UseLLVM>
+TRuntimeNode GroupGetKeysFirst(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) {
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto keyExtractor = [&](TRuntimeNode item) { return item; };
+
+ const bool isFlow = stream.GetStaticType()->IsFlow();
+
+ return pb.Chopper(stream, keyExtractor, groupSwitch, [&](TRuntimeNode key, TRuntimeNode grpItem) {
+ auto list = pb.ToList(pb.AggrConcat(
+ pb.NewOptional(pb.Concat(pb.ToString(key), pb.NewDataLiteral<NUdf::EDataSlot::String>(":"))),
+ pb.ToOptional(pb.Collect(pb.Condense1(grpItem,
+ [&] (TRuntimeNode item) { return pb.ToString(item); },
+ [&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); },
+ [&] (TRuntimeNode item, TRuntimeNode state) {
+ return pb.Concat(state, pb.ToString(item));
+ }
+ )))
+ ));
+ return isFlow ? pb.ToFlow(list) : pb.Iterator(list, {});
+ });
+}
+
+template<bool UseLLVM>
+TRuntimeNode GroupKeys(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) {
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto keyExtractor = [&](TRuntimeNode item) { return item; };
+
+ return pb.Chopper(stream, keyExtractor, groupSwitch,
+ [&](TRuntimeNode key, TRuntimeNode group) {
+ return pb.Map(pb.Take(group, pb.NewDataLiteral<ui64>(1ULL)), [&](TRuntimeNode) { return pb.ToString(key); });
+ }
+ );
+}
+
+template<bool UseLLVM>
+TRuntimeNode StreamToString(TSetup<UseLLVM>& setup, TRuntimeNode stream) {
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ stream = pb.Condense(stream, pb.NewDataLiteral<NUdf::EDataSlot::String>("|"),
+ [&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral(false); },
+ [&] (TRuntimeNode item, TRuntimeNode state) {
+ return pb.Concat(pb.Concat(state, item), pb.NewDataLiteral<NUdf::EDataSlot::String>("|"));
+ }
+ );
+ if (stream.GetStaticType()->IsFlow()) {
+ stream = pb.FromFlow(stream);
+ }
+ return stream;
+}
+
+} // unnamed
+
+Y_UNIT_TEST_SUITE(TMiniKQLChopperStreamTest) {
+ Y_UNIT_TEST_LLVM(TestEmpty) {
+ TSetup<LLVM> setup;
+
+ const auto stream = GroupWithBomb(setup, MakeStream(setup, 0U));
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGrouping) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
+ });
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingGetKeysFirst) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ stream = GroupGetKeysFirst(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
+ });
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0:0|0:01|0:0|0:0|0:0123|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingKeyNotEquals) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ return pb.NotEquals(item, key);
+ });
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*|*11*|*00*00*00*|*11*|*22*|*33*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithEmptyInput) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup, 0);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
+ });
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestSingleGroup) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ Y_UNUSED(item);
+ return pb.NewDataLiteral<bool>(false);
+ });
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*01*00*00*00*01*02*03*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithYield) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
+ stream = pb.Switch(stream,
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item1) {
+ return Group(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) {
+ Y_UNUSED(key);
+ return pb.Equals(item2, pb.NewDataLiteral<ui64>(0));
+ });
+ },
+ 1,
+ pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::String))
+ );
+
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithCutSubStreams) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+
+ stream = GroupKeys(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
+ });
+
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithYieldAndCutSubStreams) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
+ stream = pb.Switch(stream,
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item1) {
+ return GroupKeys(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) {
+ Y_UNUSED(key);
+ return pb.Equals(item2, pb.NewDataLiteral<ui64>(0));
+ });
+ },
+ 1,
+ pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::String))
+ );
+
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
+ }
+}
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 9u
+Y_UNIT_TEST_SUITE(TMiniKQLChopperFlowTest) {
+ Y_UNIT_TEST_LLVM(TestEmpty) {
+ TSetup<LLVM> setup;
+
+ const auto stream = GroupWithBomb(setup, MakeFlow(setup, 0U));
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGrouping) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeFlow(setup);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
+ });
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingGetKeysFirst) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeFlow(setup);
+ stream = GroupGetKeysFirst(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
+ });
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0:0|0:01|0:0|0:0|0:0123|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingKeyNotEquals) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeFlow(setup);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ return pb.NotEquals(item, key);
+ });
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*|*11*|*00*00*00*|*11*|*22*|*33*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithEmptyInput) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeFlow(setup, 0);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
+ });
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestSingleGroup) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeFlow(setup);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ Y_UNUSED(item);
+ return pb.NewDataLiteral<bool>(false);
+ });
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*01*00*00*00*01*02*03*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithYield) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeFlow(setup);
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
+ stream = pb.Switch(stream,
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item1) {
+ return Group(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) {
+ Y_UNUSED(key);
+ return pb.Equals(item2, pb.NewDataLiteral<ui64>(0));
+ });
+ },
+ 1,
+ pb.NewFlowType(pb.NewDataType(NUdf::EDataSlot::String))
+ );
+
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithCutSubStreams) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeFlow(setup);
+
+ stream = GroupKeys(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
+ });
+
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithYieldAndCutSubStreams) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto stream = MakeFlow(setup);
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
+ stream = pb.Switch(stream,
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item1) {
+ return GroupKeys(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) {
+ Y_UNUSED(key);
+ return pb.Equals(item2, pb.NewDataLiteral<ui64>(0));
+ });
+ },
+ 1,
+ pb.NewFlowType(pb.NewDataType(NUdf::EDataSlot::String))
+ );
+
+ const auto pgm = StreamToString(setup, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
+ }
+}
+#endif
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp
new file mode 100644
index 00000000000..72fe11dc404
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp
@@ -0,0 +1,1576 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+#include <cstring>
+#include <random>
+#include <ctime>
+#include <algorithm>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+ui64 g_Yield = std::numeric_limits<ui64>::max();
+ui64 g_TestStreamData[] = {0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2};
+ui64 g_TestYieldStreamData[] = {0, 1, 2, g_Yield, 0, g_Yield, 1, 2, 0, 1, 2, 0, g_Yield, 1, 2};
+
+template <bool WithYields>
+class TTestStreamWrapper: public TMutableComputationNode<TTestStreamWrapper<WithYields>> {
+ typedef TMutableComputationNode<TTestStreamWrapper<WithYields>> TBaseComputation;
+public:
+ class TStreamValue : public TComputationValue<TStreamValue> {
+ public:
+ using TBase = TComputationValue<TStreamValue>;
+
+ TStreamValue(TMemoryUsageInfo* memInfo, TComputationContext& compCtx, const TTestStreamWrapper* parent)
+ : TBase(memInfo)
+ , CompCtx(compCtx)
+ , Parent(parent)
+ {
+ }
+
+ private:
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
+ constexpr auto size = WithYields ? Y_ARRAY_SIZE(g_TestYieldStreamData) : Y_ARRAY_SIZE(g_TestStreamData);
+ if (Index == size) {
+ return NUdf::EFetchStatus::Finish;
+ }
+
+ const auto val = WithYields ? g_TestYieldStreamData[Index] : g_TestStreamData[Index];
+ if (g_Yield == val) {
+ ++Index;
+ return NUdf::EFetchStatus::Yield;
+ }
+
+ NUdf::TUnboxedValue* items = nullptr;
+ result = CompCtx.HolderFactory.CreateDirectArrayHolder(2, items);
+ items[0] = NUdf::TUnboxedValuePod(val);
+ if (((Index + 1) % Parent->PeakStep) == 0) {
+ auto str = MakeStringNotFilled(64ul << 20);
+ const auto& buf = str.AsStringRef();
+ memset(buf.Data(), ' ', buf.Size());
+ items[1] = std::move(str);
+ } else {
+ items[1] = NUdf::TUnboxedValuePod::Zero();
+ }
+
+ ++Index;
+ return NUdf::EFetchStatus::Ok;
+ }
+
+ private:
+ TComputationContext& CompCtx;
+ const TTestStreamWrapper* const Parent;
+ ui64 Index = 0;
+ };
+
+ TTestStreamWrapper(TComputationMutables& mutables, ui64 peakStep)
+ : TBaseComputation(mutables)
+ , PeakStep(peakStep)
+ {
+ }
+
+ NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
+ return ctx.HolderFactory.Create<TStreamValue>(ctx, this);
+ }
+
+private:
+ void RegisterDependencies() const final {
+ }
+
+private:
+ const ui64 PeakStep;
+};
+
+template <bool WithYields>
+IComputationNode* WrapTestStream(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ MKQL_ENSURE(callable.GetInputsCount() == 1, "Expected 1 args");
+ const ui64 peakStep = AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().Get<ui64>();
+ return new TTestStreamWrapper<WithYields>(ctx.Mutables, peakStep);
+}
+
+TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
+ return CreateDeterministicRandomProvider(1);
+}
+
+TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
+ return CreateDeterministicTimeProvider(10000000);
+}
+
+TComputationNodeFactory GetTestFactory() {
+ return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+ if (callable.GetType()->GetName() == "TestList") {
+ return new TExternalComputationNode(ctx.Mutables);
+ }
+ if (callable.GetType()->GetName() == "TestStream") {
+ return WrapTestStream<false>(callable, ctx);
+ }
+ if (callable.GetType()->GetName() == "TestYieldStream") {
+ return WrapTestStream<true>(callable, ctx);
+ }
+ return GetBuiltinFactory()(callable, ctx);
+ };
+}
+
+template<bool UseLLVM>
+struct TSetup_ {
+ TSetup_()
+ : Alloc(__LOCATION__)
+ {
+ FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ RandomProvider = CreateRandomProvider();
+ TimeProvider = CreateTimeProvider();
+
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
+ TAutoPtr<IComputationGraph> BuildGraph(TRuntimeNode pgm, EGraphPerProcess graphPerProcess = EGraphPerProcess::Multi, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
+ Explorer.Walk(pgm.GetNode(), *Env);
+ TComputationPatternOpts opts(Alloc.Ref(), *Env, GetTestFactory(), FunctionRegistry.Get(),
+ NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, UseLLVM ? "" : "OFF", graphPerProcess);
+ Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
+ return Pattern->Clone(opts.ToComputationOptions(*RandomProvider, *TimeProvider));
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+
+ TScopedAlloc Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
+ IComputationPattern::TPtr Pattern;
+};
+
+template <bool LLVM, bool WithYields = false>
+TRuntimeNode MakeStream(TSetup_<LLVM>& setup, ui64 peakStep) {
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TCallableBuilder callableBuilder(*setup.Env, WithYields ? "TestYieldStream" : "TestStream",
+ pb.NewStreamType(
+ pb.NewStructType({
+ {TStringBuf("a"), pb.NewDataType(NUdf::EDataSlot::Uint64)},
+ {TStringBuf("b"), pb.NewDataType(NUdf::EDataSlot::String)}
+ })
+ )
+ );
+ callableBuilder.Add(pb.NewDataLiteral(peakStep));
+
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
+template <bool OverFlow>
+TRuntimeNode Combine(TProgramBuilder& pb, TRuntimeNode stream, std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)> finishLambda) {
+ const auto keyExtractor = [&](TRuntimeNode item) {
+ return pb.Member(item, "a");
+ };
+ const auto init = [&](TRuntimeNode /*key*/, TRuntimeNode item) {
+ return item;
+ };
+ const auto update = [&](TRuntimeNode /*key*/, TRuntimeNode item, TRuntimeNode state) {
+ const auto a = pb.Add(pb.Member(item, "a"), pb.Member(state, "a"));
+ const auto b = pb.Concat(pb.Member(item, "b"), pb.Member(state, "b"));
+ return pb.NewStruct({
+ {TStringBuf("a"), a},
+ {TStringBuf("b"), b},
+ });
+ };
+
+ return OverFlow ?
+ pb.FromFlow(pb.CombineCore(pb.ToFlow(stream), keyExtractor, init, update, finishLambda, 64ul << 20)):
+ pb.CombineCore(stream, keyExtractor, init, update, finishLambda, 64ul << 20);
+}
+
+TRuntimeNode Reduce(TProgramBuilder& pb, TRuntimeNode stream) {
+ return pb.Condense(stream, pb.NewDataLiteral<ui64>(0),
+ [&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); },
+ [&] (TRuntimeNode item, TRuntimeNode state) { return pb.Add(state, item); }
+ );
+}
+
+TRuntimeNode StreamToString(TProgramBuilder& pb, TRuntimeNode stream) {
+ const auto sorted = pb.Sort(stream, pb.NewDataLiteral(true),
+ [&](TRuntimeNode item) {
+ return item;
+ });
+
+ return pb.Condense(sorted, pb.NewDataLiteral<NUdf::EDataSlot::String>("|"),
+ [&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); },
+ [&] (TRuntimeNode item, TRuntimeNode state) {
+ return pb.Concat(pb.Concat(state, pb.ToString(item)), pb.NewDataLiteral<NUdf::EDataSlot::String>("|"));
+ }
+ );
+}
+
+} // unnamed
+
+Y_UNIT_TEST_SUITE(TMiniKQLCombineStreamTest) {
+ Y_UNIT_TEST_LLVM(TestFullCombineWithOptOut) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ return pb.NewOptional(pb.Member(state, "a"));
+ };
+
+ const auto stream = MakeStream(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|4|8|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestFullCombineWithListOut) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ const auto item = pb.Member(state, "a");
+ const auto itemType = item.GetStaticType();
+ auto list = pb.NewEmptyList(itemType);
+ list = pb.Append(list, item);
+ list = pb.Append(list, item);
+ return list;
+ };
+
+ const auto stream = MakeStream(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestFullCombineWithStreamOut) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ const auto item = pb.Member(state, "a");
+ const auto itemType = item.GetStaticType();
+ auto list = pb.NewEmptyList(itemType);
+ list = pb.Append(list, item);
+ list = pb.Append(list, item);
+ return pb.Iterator(list, MakeArrayRef(&state, 1));
+ };
+
+ const auto stream = MakeStream(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestFullCombineWithOptOutAndYields) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ return pb.NewOptional(pb.Member(state, "a"));
+ };
+
+ const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|1|1|2|2|2|4|");
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFullCombineWithListAndYields) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ const auto item = pb.Member(state, "a");
+ const auto itemType = item.GetStaticType();
+ auto list = pb.NewEmptyList(itemType);
+ list = pb.Append(list, item);
+ list = pb.Append(list, item);
+ return list;
+ };
+
+ const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|0|1|1|1|1|2|2|2|2|2|2|4|4|");
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFullCombineWithStreamAndYields) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ const auto item = pb.Member(state, "a");
+ const auto itemType = item.GetStaticType();
+ auto list = pb.NewEmptyList(itemType);
+ list = pb.Append(list, item);
+ list = pb.Append(list, item);
+ return pb.Iterator(list, MakeArrayRef(&state, 1));
+ };
+
+ const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|0|1|1|1|1|2|2|2|2|2|2|4|4|");
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestPartialFlush) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ return pb.NewOptional(pb.Member(state, "a"));
+ };
+
+ const auto stream = MakeStream(setup, 6ul);
+ const auto combine = Combine<false>(pb, stream, finish);
+ {
+ const auto pgm = Reduce(pb, combine);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul);
+ }
+ {
+ const auto pgm = StreamToString(pb, combine);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|2|2|4|4|");
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestCombineInSingleProc) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ return pb.NewOptional(pb.Member(state, "a"));
+ };
+
+ const auto stream = MakeStream(setup, 6ul);
+ const auto pgm = Reduce(pb, Combine<false>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm, EGraphPerProcess::Single);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul);
+ }
+
+ Y_UNIT_TEST_LLVM(TestCombineSwithYield) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ return pb.NewOptional(pb.Member(state, "a"));
+ };
+
+ auto stream = MakeStream(setup, Max<ui64>());
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
+ stream = pb.Switch(stream,
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item) { return Combine<false>(pb, item, finish); },
+ 1,
+ pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::Uint64))
+ );
+
+ const auto pgm = StreamToString(pb, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|1|1|1|1|2|2|2|2|");
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLCombineStreamPerfTest) {
+ Y_UNIT_TEST_LLVM(TestSumDoubleBooleanKeys) {
+ TSetup_<LLVM> setup;
+
+ double positive = 0.0, negative = 0.0;
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ (sample.second > 0.0 ? positive : negative) += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}),
+ [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); },
+ [&](TRuntimeNode, TRuntimeNode item) { return item; },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); },
+ [&](TRuntimeNode, TRuntimeNode state) { return pb.NewOptional(state); },
+ 0ULL
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); });
+
+ NUdf::TUnboxedValue first, second;
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ UNIT_ASSERT_EQUAL(value.Fetch(first), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_EQUAL(value.Fetch(second), NUdf::EFetchStatus::Ok);
+ const auto t2 = TInstant::Now();
+
+ if (first.template Get<double>() > 0.0) {
+ UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), positive);
+ UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), negative);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), negative);
+ UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), positive);
+ }
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleBooleanKeys) {
+ TSetup_<LLVM> setup;
+
+ double pSum = 0.0, nSum = 0.0, pMax = 0.0, nMax = -1000.0, pMin = 1000.0, nMin = 0.0;
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ if (sample.second > 0.0) {
+ pSum += sample.second;
+ pMax = std::max(pMax, sample.second);
+ pMin = std::min(pMin, sample.second);
+ } else {
+ nSum += sample.second;
+ nMax = std::max(nMax, sample.second);
+ nMin = std::min(nMin, sample.second);
+ }
+ }
+
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}),
+ [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); },
+ [&](TRuntimeNode, TRuntimeNode item) { return pb.NewTuple({item, item, item}); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), item), pb.AggrMin(pb.Nth(state, 1U), item), pb.AggrMax(pb.Nth(state, 2U), item) }); },
+ [&](TRuntimeNode, TRuntimeNode state) { return pb.NewOptional(state); },
+ 0ULL
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); });
+
+ NUdf::TUnboxedValue first, second;
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ UNIT_ASSERT_EQUAL(value.Fetch(first), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_EQUAL(value.Fetch(second), NUdf::EFetchStatus::Ok);
+ const auto t2 = TInstant::Now();
+
+ if (first.GetElement(0).template Get<double>() > 0.0) {
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), pSum);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), pMin);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), pMax);
+
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), nSum);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), nMin);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), nMax);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), nSum);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), nMin);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), nMax);
+
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), pSum);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), pMin);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), pMax);
+ }
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleSmallKey) {
+ TSetup_<LLVM> setup;
+
+ std::unordered_map<i8, double> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ expects.emplace(sample.first, 0.0).first->second += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<i8, double>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}),
+ [&](TRuntimeNode item) { return pb.Nth(item, 0U); },
+ [&](TRuntimeNode, TRuntimeNode item) { return pb.Nth(item, 1U); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, pb.Nth(item, 1U)); },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, state})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ for (const auto& sample : I8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), ptr[i].GetElement(1).template Get<double>());
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleSmallKey) {
+ TSetup_<LLVM> setup;
+
+ std::unordered_map<i8, std::array<double, 3U>> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, std::numeric_limits<double>::max(), std::numeric_limits<double>::min()}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<i8, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}),
+ [&](TRuntimeNode item) { return pb.Nth(item, 0U); },
+ [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ for (const auto& sample : I8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleStringKey) {
+ TSetup_<LLVM> setup;
+
+ std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size());
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); });
+
+ std::unordered_map<std::string, double> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : stringI8Samples) {
+ expects.emplace(sample.first, 0.0).first->second += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::string_view, double>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}),
+ [&](TRuntimeNode item) { return pb.Nth(item, 0U); },
+ [&](TRuntimeNode, TRuntimeNode item) { return pb.Nth(item, 1U); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, pb.Nth(item, 1U)); },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, state})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items));
+ for (const auto& sample : stringI8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElements()->AsStringRef(), ptr[i].GetElement(1).template Get<double>());
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleStringKey) {
+ TSetup_<LLVM> setup;
+
+ std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size());
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); });
+
+ std::unordered_map<std::string, std::array<double, 3U>> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : stringI8Samples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::string_view, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}),
+ [&](TRuntimeNode item) { return pb.Nth(item, 0U); },
+ [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items));
+ for (const auto& sample : stringI8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElements()->AsStringRef(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumTupleKey) {
+ TSetup_<LLVM> setup;
+
+ std::vector<std::pair<std::pair<ui32, std::string>, double>> pairI8Samples(Ui16Samples.size());
+ std::transform(Ui16Samples.cbegin(), Ui16Samples.cend(), pairI8Samples.begin(), [](std::pair<ui32, double> src){ return std::make_pair(std::make_pair(ui32(src.first / 10U % 100U), ToString(src.first % 10U)), src.second); });
+
+ struct TPairHash { size_t operator()(const std::pair<ui16, std::string>& p) const { return CombineHashes(std::hash<ui32>()(p.first), std::hash<std::string_view>()(p.second)); } };
+
+ std::unordered_map<std::pair<ui32, std::string>, std::array<double, 3U>, TPairHash> expects;
+ const auto t = TInstant::Now();
+ for (const auto& sample : pairI8Samples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::pair<ui32, std::string>, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<const char*>::Id)}), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(pb.Iterator(TRuntimeNode(list, false), {}),
+ [&](TRuntimeNode item) { return pb.Nth(item, 0U); },
+ [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(pairI8Samples.size(), items));
+ for (const auto& sample : pairI8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ NUdf::TUnboxedValue* keys = nullptr;
+ pair[0] = graph->GetHolderFactory().CreateDirectArrayHolder(2U, keys);
+ keys[0] = NUdf::TUnboxedValuePod(sample.first.first);
+ keys[1] = NUdf::TUnboxedValuePod::Embedded(sample.first.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ const auto elements = ptr[i].GetElements();
+ two.emplace_back(std::make_pair(elements[0].GetElement(0).template Get<ui32>(), (elements[0].GetElements()[1]).AsStringRef()), std::array<double, 3U>{elements[1].template Get<double>(), elements[2].template Get<double>(), elements[3].template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+}
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 3u
+Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowTest) {
+ Y_UNIT_TEST_LLVM(TestFullCombineWithOptOut) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ return pb.NewOptional(pb.Member(state, "a"));
+ };
+
+ const auto stream = MakeStream(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|4|8|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestFullCombineWithListOut) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ const auto item = pb.Member(state, "a");
+ const auto itemType = item.GetStaticType();
+ auto list = pb.NewEmptyList(itemType);
+ list = pb.Append(list, item);
+ list = pb.Append(list, item);
+ return list;
+ };
+
+ const auto stream = MakeStream(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestFullCombineWithStreamOut) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ const auto item = pb.Member(state, "a");
+ const auto itemType = item.GetStaticType();
+ auto list = pb.NewEmptyList(itemType);
+ list = pb.Append(list, item);
+ list = pb.Append(list, item);
+ return pb.Iterator(list, MakeArrayRef(&state, 1));
+ };
+
+ const auto stream = MakeStream(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestFullCombineWithOptOutAndYields) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ return pb.NewOptional(pb.Member(state, "a"));
+ };
+
+ const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|1|1|2|2|2|4|");
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFullCombineWithListAndYields) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ const auto item = pb.Member(state, "a");
+ const auto itemType = item.GetStaticType();
+ auto list = pb.NewEmptyList(itemType);
+ list = pb.Append(list, item);
+ list = pb.Append(list, item);
+ return list;
+ };
+
+ const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|0|1|1|1|1|2|2|2|2|2|2|4|4|");
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFullCombineWithStreamAndYields) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ const auto item = pb.Member(state, "a");
+ const auto itemType = item.GetStaticType();
+ auto list = pb.NewEmptyList(itemType);
+ list = pb.Append(list, item);
+ list = pb.Append(list, item);
+ return pb.Iterator(list, MakeArrayRef(&state, 1));
+ };
+
+ const auto stream = MakeStream<LLVM, true>(setup, Max<ui64>());
+ const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Yield);
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|0|1|1|1|1|2|2|2|2|2|2|4|4|");
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestPartialFlush) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ return pb.NewOptional(pb.Member(state, "a"));
+ };
+
+ const auto stream = MakeStream(setup, 6ul);
+ const auto combine = Combine<true>(pb, stream, finish);
+ {
+ const auto pgm = Reduce(pb, combine);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul);
+ }
+ {
+ const auto pgm = StreamToString(pb, combine);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|2|2|4|4|");
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestCombineInSingleProc) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ return pb.NewOptional(pb.Member(state, "a"));
+ };
+
+ const auto stream = MakeStream(setup, 6ul);
+ const auto pgm = Reduce(pb, Combine<true>(pb, stream, finish));
+ const auto graph = setup.BuildGraph(pgm, EGraphPerProcess::Single);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul);
+ }
+
+ Y_UNIT_TEST_LLVM(TestCombineSwithYield) {
+ TSetup_<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
+ return pb.NewOptional(pb.Member(state, "a"));
+ };
+
+ auto stream = MakeStream(setup, Max<ui64>());
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
+ stream = pb.Switch(stream,
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item) { return Combine<true>(pb, item, finish); },
+ 1,
+ pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::Uint64))
+ );
+
+ const auto pgm = StreamToString(pb, stream);
+ const auto graph = setup.BuildGraph(pgm);
+ const auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|1|1|1|1|2|2|2|2|");
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowPerfTest) {
+ Y_UNIT_TEST_LLVM(TestSumDoubleBooleanKeys) {
+ TSetup_<LLVM> setup;
+
+ double positive = 0.0, negative = 0.0;
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ (sample.second > 0.0 ? positive : negative) += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.FromFlow(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); },
+ [&](TRuntimeNode, TRuntimeNode item) { return item; },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); },
+ [&](TRuntimeNode, TRuntimeNode state) { return pb.NewOptional(state); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); });
+
+ NUdf::TUnboxedValue first, second;
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ UNIT_ASSERT_EQUAL(value.Fetch(first), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_EQUAL(value.Fetch(second), NUdf::EFetchStatus::Ok);
+ const auto t2 = TInstant::Now();
+
+ if (first.template Get<double>() > 0.0) {
+ UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), positive);
+ UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), negative);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), negative);
+ UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), positive);
+ }
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleBooleanKeys) {
+ TSetup_<LLVM> setup;
+
+ double pSum = 0.0, nSum = 0.0, pMax = 0.0, nMax = -1000.0, pMin = 1000.0, nMin = 0.0;
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ if (sample.second > 0.0) {
+ pSum += sample.second;
+ pMax = std::max(pMax, sample.second);
+ pMin = std::min(pMin, sample.second);
+ } else {
+ nSum += sample.second;
+ nMax = std::max(nMax, sample.second);
+ nMin = std::min(nMin, sample.second);
+ }
+ }
+
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.FromFlow(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); },
+ [&](TRuntimeNode, TRuntimeNode item) { return pb.NewTuple({item, item, item}); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), item), pb.AggrMin(pb.Nth(state, 1U), item), pb.AggrMax(pb.Nth(state, 2U), item) }); },
+ [&](TRuntimeNode, TRuntimeNode state) { return pb.NewOptional(state); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); });
+
+ NUdf::TUnboxedValue first, second;
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ UNIT_ASSERT_EQUAL(value.Fetch(first), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_EQUAL(value.Fetch(second), NUdf::EFetchStatus::Ok);
+ const auto t2 = TInstant::Now();
+
+ if (first.GetElement(0).template Get<double>() > 0.0) {
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), pSum);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), pMin);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), pMax);
+
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), nSum);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), nMin);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), nMax);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), nSum);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), nMin);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), nMax);
+
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), pSum);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), pMin);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), pMax);
+ }
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleSmallKey) {
+ TSetup_<LLVM> setup;
+
+ std::unordered_map<i8, double> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ expects.emplace(sample.first, 0.0).first->second += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<i8, double>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.Nth(item, 0U); },
+ [&](TRuntimeNode, TRuntimeNode item) { return pb.Nth(item, 1U); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, pb.Nth(item, 1U)); },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, state})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ for (const auto& sample : I8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), ptr[i].GetElement(1).template Get<double>());
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleSmallKey) {
+ TSetup_<LLVM> setup;
+
+ std::unordered_map<i8, std::array<double, 3U>> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, std::numeric_limits<double>::max(), std::numeric_limits<double>::min()}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<i8, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.Nth(item, 0U); },
+ [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ for (const auto& sample : I8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleStringKey) {
+ TSetup_<LLVM> setup;
+
+ std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size());
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); });
+
+ std::unordered_map<std::string, double> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : stringI8Samples) {
+ expects.emplace(sample.first, 0.0).first->second += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::string_view, double>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.Nth(item, 0U); },
+ [&](TRuntimeNode, TRuntimeNode item) { return pb.Nth(item, 1U); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, pb.Nth(item, 1U)); },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, state})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items));
+ for (const auto& sample : stringI8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElements()->AsStringRef(), ptr[i].GetElement(1).template Get<double>());
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleStringKey) {
+ TSetup_<LLVM> setup;
+
+ std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size());
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); });
+
+ std::unordered_map<std::string, std::array<double, 3U>> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : stringI8Samples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::string_view, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.Nth(item, 0U); },
+ [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items));
+ for (const auto& sample : stringI8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElements()->AsStringRef(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumTupleKey) {
+ TSetup_<LLVM> setup;
+
+ std::vector<std::pair<std::pair<ui32, std::string>, double>> pairI8Samples(Ui16Samples.size());
+ std::transform(Ui16Samples.cbegin(), Ui16Samples.cend(), pairI8Samples.begin(), [](std::pair<ui16, double> src){ return std::make_pair(std::make_pair(ui32(src.first / 10U % 100U), ToString(src.first % 10U)), src.second); });
+
+ struct TPairHash { size_t operator()(const std::pair<ui32, std::string>& p) const { return CombineHashes(std::hash<ui32>()(p.first), std::hash<std::string_view>()(p.second)); } };
+
+ std::unordered_map<std::pair<ui32, std::string>, std::array<double, 3U>, TPairHash> expects;
+ const auto t = TInstant::Now();
+ for (const auto& sample : pairI8Samples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::pair<ui32, std::string>, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<const char*>::Id)}), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.Nth(item, 0U); },
+ [&](TRuntimeNode, TRuntimeNode item) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({v, v, v}); },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) { const auto v = pb.Nth(item, 1U); return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), v), pb.AggrMin(pb.Nth(state, 1U), v), pb.AggrMax(pb.Nth(state, 2U), v)}); },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({key, pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Nth(state, 2U)})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(pairI8Samples.size(), items));
+ for (const auto& sample : pairI8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ NUdf::TUnboxedValue* keys = nullptr;
+ pair[0] = graph->GetHolderFactory().CreateDirectArrayHolder(2U, keys);
+ keys[0] = NUdf::TUnboxedValuePod(sample.first.first);
+ keys[1] = NUdf::TUnboxedValuePod::Embedded(sample.first.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ const auto elements = ptr[i].GetElements();
+ two.emplace_back(std::make_pair(elements[0].GetElement(0).template Get<ui32>(), (elements[0].GetElements()[1]).AsStringRef()), std::array<double, 3U>{elements[1].template Get<double>(), elements[2].template Get<double>(), elements[3].template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ const auto border = 9124596000000000ULL;
+
+ Y_UNIT_TEST_LLVM(TestTpch) {
+ TSetup_<LLVM> setup;
+
+ struct TPairHash { size_t operator()(const std::pair<std::string_view, std::string_view>& p) const { return CombineHashes(std::hash<std::string_view>()(p.first), std::hash<std::string_view>()(p.second)); } };
+
+ std::unordered_map<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>, TPairHash> expects;
+ const auto t = TInstant::Now();
+ for (auto& sample : TpchSamples) {
+ if (std::get<0U>(sample) <= border) {
+ const auto& ins = expects.emplace(std::pair<std::string_view, std::string_view>{std::get<1U>(sample), std::get<2U>(sample)}, std::pair<ui64, std::array<double, 5U>>{0ULL, {0., 0., 0., 0., 0.}});
+ auto& item = ins.first->second;
+ ++item.first;
+ std::get<0U>(item.second) += std::get<3U>(sample);
+ std::get<1U>(item.second) += std::get<5U>(sample);
+ std::get<2U>(item.second) += std::get<6U>(sample);
+ const auto v = std::get<3U>(sample) * (1. - std::get<5U>(sample));
+ std::get<3U>(item.second) += v;
+ std::get<4U>(item.second) += v * (1. + std::get<4U>(sample));
+ }
+ }
+ for (auto& item : expects) {
+ std::get<1U>(item.second.second) /= item.second.first;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::pair<std::string, std::string>, std::pair<ui64, std::array<double, 5U>>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui64>::Id),
+ pb.NewDataType(NUdf::TDataType<const char*>::Id),
+ pb.NewDataType(NUdf::TDataType<const char*>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id)
+ }));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.CombineCore(
+ pb.Map(pb.Filter(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.AggrLessOrEqual(pb.Nth(item, 0U), pb.NewDataLiteral<ui64>(border)); }
+ ),
+ [&](TRuntimeNode item) { return pb.NewTuple({pb.Nth(item, 1U), pb.Nth(item, 2U),pb.Nth(item, 3U),pb.Nth(item, 4U),pb.Nth(item, 5U),pb.Nth(item, 6U)}); } ),
+ [&](TRuntimeNode item) { return pb.NewTuple({pb.Nth(item, 0U), pb.Nth(item, 1U)}); },
+ [&](TRuntimeNode, TRuntimeNode item) {
+ const auto price = pb.Nth(item, 2U);
+ const auto disco = pb.Nth(item, 4U);
+ const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco));
+ return pb.NewTuple({pb.NewDataLiteral<ui64>(1ULL), price, disco, pb.Nth(item, 5U), v, pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), pb.Nth(item, 3U))) });
+ },
+ [&](TRuntimeNode, TRuntimeNode item, TRuntimeNode state) {
+ const auto price = pb.Nth(item, 2U);
+ const auto disco = pb.Nth(item, 4U);
+ const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco));
+ return pb.NewTuple({pb.Increment(pb.Nth(state, 0U)), pb.AggrAdd(pb.Nth(state, 1U), price), pb.AggrAdd(pb.Nth(state, 2U), disco), pb.AggrAdd(pb.Nth(state, 3U), pb.Nth(item, 5U)), pb.AggrAdd(pb.Nth(state, 4U), v), pb.AggrAdd(pb.Nth(state, 5U), pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), pb.Nth(item, 3U)))) });
+ },
+ [&](TRuntimeNode key, TRuntimeNode state) { return pb.NewOptional(pb.NewTuple({pb.Nth(key, 0U), pb.Nth(key, 1U), pb.Nth(state, 0U), pb.Nth(state, 1U), pb.Div(pb.Nth(state, 2U), pb.Nth(state, 0U)), pb.Nth(state, 3U), pb.Nth(state, 4U), pb.Nth(state, 5U)})); },
+ 0ULL
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, EGraphPerProcess::Multi, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(TpchSamples.size(), items));
+ for (const auto& sample : TpchSamples) {
+ NUdf::TUnboxedValue* elements = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(7U, elements);
+ elements[0] = NUdf::TUnboxedValuePod(std::get<0U>(sample));
+ elements[1] = NUdf::TUnboxedValuePod::Embedded(std::get<1U>(sample));
+ elements[2] = NUdf::TUnboxedValuePod::Embedded(std::get<2U>(sample));
+ elements[3] = NUdf::TUnboxedValuePod(std::get<3U>(sample));
+ elements[4] = NUdf::TUnboxedValuePod(std::get<4U>(sample));
+ elements[5] = NUdf::TUnboxedValuePod(std::get<5U>(sample));
+ elements[6] = NUdf::TUnboxedValuePod(std::get<6U>(sample));
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ const auto elements = ptr[i].GetElements();
+ two.emplace_back(std::make_pair(elements[0].AsStringRef(), elements[1].AsStringRef()), std::pair<ui64, std::array<double, 5U>>{elements[2].template Get<ui64>(), {elements[3].template Get<double>(), elements[4].template Get<double>(), elements[5].template Get<double>(), elements[6].template Get<double>(), elements[7].template Get<double>()}});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+}
+#endif
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp
new file mode 100644
index 00000000000..05f1e5bd882
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_compare_ut.cpp
@@ -0,0 +1,1119 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_printer.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_list_adapter.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+#include <cfloat>
+#include <utility>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLCompareTest) {
+ Y_UNIT_TEST_LLVM(SqlString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id);
+ auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("010"));
+ auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("020"));
+
+ auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ auto optionalType = pb.NewOptionalType(dataType);
+ auto pairType = pb.NewTupleType({optionalType, optionalType});
+ auto list = pb.NewList(pairType, {
+ pb.NewTuple({data0, data0}),
+ pb.NewTuple({data0, data1}),
+ pb.NewTuple({data0, data2}),
+
+ pb.NewTuple({data1, data0}),
+ pb.NewTuple({data1, data1}),
+ pb.NewTuple({data1, data2}),
+
+ pb.NewTuple({data2, data0}),
+ pb.NewTuple({data2, data1}),
+ pb.NewTuple({data2, data2})
+ });
+
+ auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1))
+ });
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0)); // ==
+ UNIT_ASSERT(!item.GetElement(1)); // !=
+ UNIT_ASSERT(!item.GetElement(2)); // <
+ UNIT_ASSERT(!item.GetElement(3)); // <=
+ UNIT_ASSERT(!item.GetElement(4)); // >
+ UNIT_ASSERT(!item.GetElement(5)); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0)); // ==
+ UNIT_ASSERT(!item.GetElement(1)); // !=
+ UNIT_ASSERT(!item.GetElement(2)); // <
+ UNIT_ASSERT(!item.GetElement(3)); // <=
+ UNIT_ASSERT(!item.GetElement(4)); // >
+ UNIT_ASSERT(!item.GetElement(5)); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0)); // ==
+ UNIT_ASSERT(!item.GetElement(1)); // !=
+ UNIT_ASSERT(!item.GetElement(2)); // <
+ UNIT_ASSERT(!item.GetElement(3)); // <=
+ UNIT_ASSERT(!item.GetElement(4)); // >
+ UNIT_ASSERT(!item.GetElement(5)); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0)); // ==
+ UNIT_ASSERT(!item.GetElement(1)); // !=
+ UNIT_ASSERT(!item.GetElement(2)); // <
+ UNIT_ASSERT(!item.GetElement(3)); // <=
+ UNIT_ASSERT(!item.GetElement(4)); // >
+ UNIT_ASSERT(!item.GetElement(5)); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0)); // ==
+ UNIT_ASSERT(!item.GetElement(1)); // !=
+ UNIT_ASSERT(!item.GetElement(2)); // <
+ UNIT_ASSERT(!item.GetElement(3)); // <=
+ UNIT_ASSERT(!item.GetElement(4)); // >
+ UNIT_ASSERT(!item.GetElement(5)); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(AggrString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id);
+ auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("010"));
+ auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("020"));
+
+ auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ auto optionalType = pb.NewOptionalType(dataType);
+ auto pairType = pb.NewTupleType({optionalType, optionalType});
+ auto list = pb.NewList(pairType, {
+ pb.NewTuple({data0, data0}),
+ pb.NewTuple({data0, data1}),
+ pb.NewTuple({data0, data2}),
+
+ pb.NewTuple({data1, data0}),
+ pb.NewTuple({data1, data1}),
+ pb.NewTuple({data1, data2}),
+
+ pb.NewTuple({data2, data0}),
+ pb.NewTuple({data2, data1}),
+ pb.NewTuple({data2, data2})
+ });
+
+ auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.AggrEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrNotEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrLess(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrLessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrGreater(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrGreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1))
+ });
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(SqlFloats) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data1 = pb.NewDataLiteral<float>(-7.0f);
+ auto data2 = pb.NewDataLiteral<float>(3.0f);
+ auto data3 = pb.NewDataLiteral<float>(0.0f*HUGE_VALF);
+ auto data4 = pb.NewDataLiteral<double>(-7.0);
+ auto data5 = pb.NewDataLiteral<double>(3.0);
+ auto data6 = pb.NewDataLiteral<double>(0.0*HUGE_VAL);
+
+ auto pairType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<float>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)});
+ auto list = pb.NewList(pairType, {
+ pb.NewTuple({data1, data4}),
+ pb.NewTuple({data1, data5}),
+ pb.NewTuple({data1, data6}),
+
+ pb.NewTuple({data2, data4}),
+ pb.NewTuple({data2, data5}),
+ pb.NewTuple({data2, data6}),
+
+ pb.NewTuple({data3, data4}),
+ pb.NewTuple({data3, data5}),
+ pb.NewTuple({data3, data6}),
+ });
+
+ auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1))
+ });
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(AggrFloats) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data1 = pb.NewDataLiteral<float>(-7.0f);
+ auto data2 = pb.NewDataLiteral<float>(3.0f);
+ auto data3 = pb.NewDataLiteral<float>(0.0f*HUGE_VALF);
+
+ auto pairType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<float>::Id), pb.NewDataType(NUdf::TDataType<float>::Id)});
+ auto list = pb.NewList(pairType, {
+ pb.NewTuple({data1, data1}),
+ pb.NewTuple({data1, data2}),
+ pb.NewTuple({data1, data3}),
+
+ pb.NewTuple({data2, data1}),
+ pb.NewTuple({data2, data2}),
+ pb.NewTuple({data2, data3}),
+
+ pb.NewTuple({data3, data1}),
+ pb.NewTuple({data3, data2}),
+ pb.NewTuple({data3, data3}),
+ });
+
+ auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.AggrEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrNotEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrLess(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrLessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrGreater(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrGreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1))
+ });
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(SqlSigned) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data1 = pb.NewDataLiteral<i16>(-1);
+ auto data2 = pb.NewDataLiteral<i16>(+1);
+ auto data3 = pb.NewDataLiteral<i32>(-1);
+ auto data4 = pb.NewDataLiteral<i32>(+1);
+
+ auto pairType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i16>::Id), pb.NewDataType(NUdf::TDataType<i32>::Id)});
+ auto list = pb.NewList(pairType, {
+ pb.NewTuple({data1, data3}),
+ pb.NewTuple({data1, data4}),
+
+ pb.NewTuple({data2, data3}),
+ pb.NewTuple({data2, data4})
+ });
+
+ auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1))
+ });
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(SqlSignedAndUnsigned) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data1 = pb.NewDataLiteral<i8>(-1);
+ auto data2 = pb.NewDataLiteral<i8>(127);
+ auto data3 = pb.NewDataLiteral<ui8>(0xFF);
+ auto data4 = pb.NewDataLiteral<ui8>(127);
+
+ auto pairType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<ui8>::Id)});
+ auto list = pb.NewList(pairType, {
+ pb.NewTuple({data1, data3}),
+ pb.NewTuple({data1, data4}),
+
+ pb.NewTuple({data2, data3}),
+ pb.NewTuple({data2, data4})
+ });
+
+ auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1))
+ });
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(SqlUnsignedAndSigned) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data1 = pb.NewDataLiteral<ui16>(0);
+ auto data2 = pb.NewDataLiteral<ui16>(0xFFFF);
+ auto data3 = pb.NewDataLiteral<i64>(-1);
+ auto data4 = pb.NewDataLiteral<i64>(0xFFFF);
+
+ auto pairType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui16>::Id), pb.NewDataType(NUdf::TDataType<i64>::Id)});
+ auto list = pb.NewList(pairType, {
+ pb.NewTuple({data1, data3}),
+ pb.NewTuple({data1, data4}),
+
+ pb.NewTuple({data2, data3}),
+ pb.NewTuple({data2, data4})
+ });
+
+ auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1))
+ });
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(SimpleSqlString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto opt1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ auto opt2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ auto optEmpty = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id);
+ auto pgmReturn = pb.NewEmptyList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id)));
+
+ pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1, opt1));
+ pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1, optEmpty));
+ pgmReturn = pb.Append(pgmReturn, pb.Equals(optEmpty, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.Equals(optEmpty, optEmpty));
+
+ pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1, opt1));
+ pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1, optEmpty));
+ pgmReturn = pb.Append(pgmReturn, pb.NotEquals(optEmpty, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.NotEquals(optEmpty, optEmpty));
+
+ pgmReturn = pb.Append(pgmReturn, pb.Less(opt1, opt1));
+ pgmReturn = pb.Append(pgmReturn, pb.Less(opt1, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.Less(opt2, opt1));
+ pgmReturn = pb.Append(pgmReturn, pb.Less(opt1, optEmpty));
+ pgmReturn = pb.Append(pgmReturn, pb.Less(optEmpty, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.Less(optEmpty, optEmpty));
+
+ pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(opt1, opt1));
+ pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(opt1, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(opt2, opt1));
+ pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(opt1, optEmpty));
+ pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(optEmpty, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.LessOrEqual(optEmpty, optEmpty));
+
+ pgmReturn = pb.Append(pgmReturn, pb.Greater(opt1, opt1));
+ pgmReturn = pb.Append(pgmReturn, pb.Greater(opt1, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.Greater(opt2, opt1));
+ pgmReturn = pb.Append(pgmReturn, pb.Greater(opt1, optEmpty));
+ pgmReturn = pb.Append(pgmReturn, pb.Greater(optEmpty, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.Greater(optEmpty, optEmpty));
+
+ pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(opt1, opt1));
+ pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(opt1, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(opt2, opt1));
+ pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(opt1, optEmpty));
+ pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(optEmpty, opt2));
+ pgmReturn = pb.Append(pgmReturn, pb.GreaterOrEqual(optEmpty, optEmpty));
+
+ auto opt1s = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("A"));
+ auto opt2s = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("B"));
+ auto optEmptys = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id);
+
+ pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1s, opt1s));
+ pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1s, opt2s));
+ pgmReturn = pb.Append(pgmReturn, pb.Equals(opt1s, optEmptys));
+ pgmReturn = pb.Append(pgmReturn, pb.Equals(optEmptys, opt2s));
+ pgmReturn = pb.Append(pgmReturn, pb.Equals(optEmptys, optEmptys));
+
+ pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1s, opt1s));
+ pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1s, opt2s));
+ pgmReturn = pb.Append(pgmReturn, pb.NotEquals(opt1s, optEmptys));
+ pgmReturn = pb.Append(pgmReturn, pb.NotEquals(optEmptys, opt2s));
+ pgmReturn = pb.Append(pgmReturn, pb.NotEquals(optEmptys, optEmptys));
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ // equals
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // not equals
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // less
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // less or equal
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // greater
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // greater or equal
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // equals - string
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // not equals - string
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TzMin) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui16>(1ULL);
+ const auto data2 = pb.NewDataLiteral<ui16>(7ULL);
+
+ const auto data3 = pb.NewDataLiteral<ui32>(1ULL);
+ const auto data4 = pb.NewDataLiteral<ui32>(7ULL);
+
+ const auto zones = pb.ListFromRange(data1, data2, data1);
+ const auto dates = pb.ListFromRange(data3, data4, data3);
+
+ const auto source = pb.Map(pb.Zip({pb.Reverse(dates), zones}),
+ [&](TRuntimeNode item) {
+ return pb.AddTimezone(pb.ToIntegral(pb.Nth(item, 0U), pb.NewDataType(NUdf::EDataSlot::Datetime, true)), pb.Nth(item, 1U));
+ });
+
+ const auto pgmReturn = pb.ToString(pb.Unwrap(pb.Fold1(source,
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.Min(item, state); }
+ ), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), __FILE__, __LINE__, 0));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNBOXED_VALUE_STR_EQUAL(result, "1970-01-01T03:00:01,Africa/Asmara");
+ }
+
+ Y_UNIT_TEST_LLVM(TzMax) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui16>(1ULL);
+ const auto data2 = pb.NewDataLiteral<ui16>(7ULL);
+
+ const auto data3 = pb.NewDataLiteral<ui32>(1ULL);
+ const auto data4 = pb.NewDataLiteral<ui32>(7ULL);
+
+ const auto zones = pb.ListFromRange(data1, data2, data1);
+ const auto dates = pb.ListFromRange(data3, data4, data3);
+
+ const auto source = pb.Map(pb.Zip({dates, pb.Reverse(zones)}),
+ [&](TRuntimeNode item) {
+ return pb.AddTimezone(pb.ToIntegral(pb.Nth(item, 0U), pb.NewDataType(NUdf::EDataSlot::Datetime, true)), pb.Nth(item, 1U));
+ });
+
+ const auto pgmReturn = pb.ToString(pb.Unwrap(pb.Fold1(source,
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.Max(item, state); }
+ ), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), __FILE__, __LINE__, 0));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNBOXED_VALUE_STR_EQUAL(result, "1970-01-01T03:00:06,Europe/Moscow");
+ }
+
+ Y_UNIT_TEST_LLVM(TzAggrMin) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui16>(1ULL);
+ const auto data2 = pb.NewDataLiteral<ui16>(7ULL);
+
+ const auto data3 = pb.NewDataLiteral<ui32>(1ULL);
+ const auto data4 = pb.NewDataLiteral<ui32>(7ULL);
+
+ const auto zones = pb.ListFromRange(data1, data2, data1);
+ const auto dates = pb.ListFromRange(data3, data4, data3);
+
+ const auto source = pb.FlatMap(zones,
+ [&](TRuntimeNode zone) {
+ return pb.Map(dates,
+ [&](TRuntimeNode date) {
+ return pb.AddTimezone(pb.ToIntegral(date, pb.NewDataType(NUdf::EDataSlot::Datetime, true)), zone);
+ });
+ });
+
+ const auto pgmReturn = pb.ToString(pb.Unwrap(pb.Fold1(source,
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrMin(item, state); }
+ ), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), __FILE__, __LINE__, 0));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNBOXED_VALUE_STR_EQUAL(result, "1970-01-01T03:00:01,Europe/Moscow");
+ }
+
+ Y_UNIT_TEST_LLVM(TzAggrMax) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui16>(1ULL);
+ const auto data2 = pb.NewDataLiteral<ui16>(7ULL);
+
+ const auto data3 = pb.NewDataLiteral<ui32>(1ULL);
+ const auto data4 = pb.NewDataLiteral<ui32>(7ULL);
+
+ const auto zones = pb.ListFromRange(data1, data2, data1);
+ const auto dates = pb.ListFromRange(data3, data4, data3);
+
+ const auto source = pb.FlatMap(dates,
+ [&](TRuntimeNode date) {
+ return pb.Map(zones,
+ [&](TRuntimeNode zone) {
+ return pb.AddTimezone(pb.ToIntegral(date, pb.NewDataType(NUdf::EDataSlot::Datetime, true)), zone);
+ });
+ });
+
+ const auto pgmReturn = pb.ToString(pb.Unwrap(pb.Fold1(source,
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrMax(item, state); }
+ ), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), __FILE__, __LINE__, 0));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNBOXED_VALUE_STR_EQUAL(result, "1970-01-01T03:00:06,Africa/Asmara");
+ }
+
+ Y_UNIT_TEST_LLVM(TestAggrMinMaxFloats) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<float>(0.0f*HUGE_VALF);
+ const auto data2 = pb.NewDataLiteral<float>(HUGE_VALF);
+ const auto data3 = pb.NewDataLiteral<float>(3.14f);
+ const auto data4 = pb.NewDataLiteral<float>(-2.13f);
+ const auto data5 = pb.NewDataLiteral<float>(-HUGE_VALF);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5});
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode left) {
+ return pb.Map(list,
+ [&](TRuntimeNode right) {
+ return pb.NewTuple({pb.AggrMin(left, right), pb.AggrMax(left, right)});
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::isnan(item.GetElement(0).Get<float>()));
+ UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>()));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), HUGE_VALF);
+ UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>()));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), 3.14f);
+ UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>()));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f);
+ UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>()));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>()));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), HUGE_VALF);
+ UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>()));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), 3.14f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), 3.14f);
+ UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>()));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), 3.14f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), 3.14f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f);
+ UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>()));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), -2.13f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), -2.13f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT(std::isnan(item.GetElement(1).Get<float>()));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), -2.13f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).Get<float>(), -HUGE_VALF);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp
new file mode 100644
index 00000000000..eefce3227a2
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.cpp
@@ -0,0 +1,4799 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_printer.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+#include <cfloat>
+#include <utility>
+#include <random>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+constexpr auto TotalSambles =
+#ifndef NDEBUG
+333333U;
+#else
+33333333ULL;
+#endif
+
+}
+
+std::vector<std::pair<i8, double>> MakeSamples() {
+ std::default_random_engine eng;
+ std::uniform_int_distribution<i8> keys(-100, +100);
+ std::uniform_real_distribution<double> unif(-999.0, +999.0);
+
+ std::vector<std::pair<i8, double>> samples(TotalSambles);
+
+ eng.seed(std::time(nullptr));
+ std::generate(samples.begin(), samples.end(), std::bind(&std::make_pair<i8, double>, std::bind(std::move(keys), std::move(eng)), std::bind(std::move(unif), std::move(eng))));
+ return samples;
+}
+
+std::vector<std::pair<ui16, double>> MakeOtherSamples() {
+ std::default_random_engine eng;
+ std::uniform_int_distribution<ui16> keys(0U, 65535U);
+ std::uniform_real_distribution<double> unif(-999.0, +999.0);
+
+ std::vector<std::pair<ui16, double>> samples(TotalSambles);
+
+ eng.seed(std::time(nullptr));
+ std::generate(samples.begin(), samples.end(), std::bind(&std::make_pair<ui16, double>, std::bind(std::move(keys), std::move(eng)), std::bind(std::move(unif), std::move(eng))));
+ return samples;
+}
+
+std::vector<std::tuple<ui64, std::string, std::string, double, double, double, double>> MakeTpchSamples() {
+ std::default_random_engine eng;
+ std::uniform_int_distribution<ui64> dates(694303200000000ULL, 9124596000000005ULL);
+ std::uniform_int_distribution<ui8> keys(0U, 3U);
+ std::uniform_real_distribution<double> prices(900., 105000.0);
+ std::uniform_real_distribution<double> taxes(0., 0.08);
+ std::uniform_real_distribution<double> discs(0., 0.1);
+ std::uniform_real_distribution<double> qntts(1., 50.);
+
+ std::random_device rd;
+ std::mt19937 gen(rd());
+
+ std::vector<std::tuple<ui64, std::string, std::string, double, double, double, double>> samples(TotalSambles);
+
+ eng.seed(std::time(nullptr));
+
+ std::generate(samples.begin(), samples.end(), [&]() {
+ switch(keys(gen)) {
+ case 0U: return std::make_tuple(dates(gen), "N", "O", prices(gen), taxes(gen), discs(gen), qntts(gen));
+ case 1U: return std::make_tuple(dates(gen), "A", "F", prices(gen), taxes(gen), discs(gen), qntts(gen));
+ case 2U: return std::make_tuple(dates(gen), "N", "F", prices(gen), taxes(gen), discs(gen), qntts(gen));
+ case 3U: return std::make_tuple(dates(gen), "R", "F", prices(gen), taxes(gen), discs(gen), qntts(gen));
+ }
+ Y_FAIL("Unexpected");
+ });
+ return samples;
+}
+
+extern const std::vector<std::pair<i8, double>> I8Samples = MakeSamples();
+extern const std::vector<std::pair<ui16, double>> Ui16Samples = MakeOtherSamples();
+extern const std::vector<std::tuple<ui64, std::string, std::string, double, double, double, double>> TpchSamples = MakeTpchSamples();
+
+Y_UNIT_TEST_SUITE(TMiniKQLComputationNodeTest) {
+ Y_UNIT_TEST_LLVM(TestEmbeddedByteAt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui32>(0);
+ const auto data1 = pb.NewDataLiteral<ui32>(3);
+ const auto data2 = pb.NewDataLiteral<ui32>(9);
+ const auto data3 = pb.NewDataLiteral<ui32>(13);
+ const auto data4 = pb.NewDataLiteral<ui32>(Max<ui32>());
+
+ const auto str = pb.NewDataLiteral<NUdf::EDataSlot::String>("0123456789AB");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ByteAt(str, item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x30);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x33);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x39);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestStringByteAt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui32>(0);
+ const auto data1 = pb.NewDataLiteral<ui32>(3);
+ const auto data2 = pb.NewDataLiteral<ui32>(21);
+ const auto data3 = pb.NewDataLiteral<ui32>(22);
+ const auto data4 = pb.NewDataLiteral<ui32>(Max<ui32>());
+
+ const auto str = pb.NewDataLiteral<NUdf::EDataSlot::String>("0123456789ABCDEFGHIJKL");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ByteAt(str, item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x30);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x33);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0x4C);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCountBitsUI8) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui8>(0x00);
+ const auto data1 = pb.NewDataLiteral<ui8>(0x01);
+ const auto data2 = pb.NewDataLiteral<ui8>(0x10);
+ const auto data3 = pb.NewDataLiteral<ui8>(0x13);
+ const auto data4 = pb.NewDataLiteral<ui8>(0xF0);
+ const auto data5 = pb.NewDataLiteral<ui8>(0x0F);
+ const auto data6 = pb.NewDataLiteral<ui8>(0x55);
+ const auto data7 = pb.NewDataLiteral<ui8>(0xAA);
+ const auto data8 = pb.NewDataLiteral<ui8>(0x7F);
+ const auto data9 = pb.NewDataLiteral<ui8>(0xFF);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui8>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.CountBits(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 8);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCountBitsUI16) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui16>(0x0000);
+ const auto data1 = pb.NewDataLiteral<ui16>(0x0100);
+ const auto data2 = pb.NewDataLiteral<ui16>(0x0010);
+ const auto data3 = pb.NewDataLiteral<ui16>(0x0130);
+ const auto data4 = pb.NewDataLiteral<ui16>(0xF000);
+ const auto data5 = pb.NewDataLiteral<ui16>(0x0F00);
+ const auto data6 = pb.NewDataLiteral<ui16>(0x0550);
+ const auto data7 = pb.NewDataLiteral<ui16>(0xA00A);
+ const auto data8 = pb.NewDataLiteral<ui16>(0x700F);
+ const auto data9 = pb.NewDataLiteral<ui16>(0x0FF0);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.CountBits(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 8);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestShiftLeft) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui8>(0);
+ const auto data1 = pb.NewDataLiteral<ui8>(1);
+ const auto data2 = pb.NewDataLiteral<ui8>(7);
+ const auto data3 = pb.NewDataLiteral<ui8>(10);
+ const auto data4 = pb.NewDataLiteral<ui8>(30);
+ const auto data5 = pb.NewDataLiteral<ui8>(32);
+ const auto data6 = pb.NewDataLiteral<ui8>(40);
+ const auto data7 = pb.NewDataLiteral<ui8>(57);
+ const auto data8 = pb.NewDataLiteral<ui8>(200);
+ const auto data9 = pb.NewDataLiteral<ui8>(255);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui8>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ShiftLeft(pb.NewDataLiteral<ui32>(0x830500F1U), item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x830500F1U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x060A01E2U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x82807880U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x1403C400U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x40000000U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestShiftRight) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui8>(0);
+ const auto data1 = pb.NewDataLiteral<ui8>(1);
+ const auto data2 = pb.NewDataLiteral<ui8>(7);
+ const auto data3 = pb.NewDataLiteral<ui8>(10);
+ const auto data4 = pb.NewDataLiteral<ui8>(30);
+ const auto data5 = pb.NewDataLiteral<ui8>(32);
+ const auto data6 = pb.NewDataLiteral<ui8>(40);
+ const auto data7 = pb.NewDataLiteral<ui8>(57);
+ const auto data8 = pb.NewDataLiteral<ui8>(200);
+ const auto data9 = pb.NewDataLiteral<ui8>(255);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui8>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ShiftRight(pb.NewDataLiteral<ui32>(0x830500F1U), item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x830500F1U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x41828078U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x01060A01U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x20C140U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0x2U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestRotLeft) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui8>(0);
+ const auto data1 = pb.NewDataLiteral<ui8>(1);
+ const auto data2 = pb.NewDataLiteral<ui8>(7);
+ const auto data3 = pb.NewDataLiteral<ui8>(10);
+ const auto data4 = pb.NewDataLiteral<ui8>(30);
+ const auto data5 = pb.NewDataLiteral<ui8>(32);
+ const auto data6 = pb.NewDataLiteral<ui8>(40);
+ const auto data7 = pb.NewDataLiteral<ui8>(57);
+ const auto data8 = pb.NewDataLiteral<ui8>(200);
+ const auto data9 = pb.NewDataLiteral<ui8>(255);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui8>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.RotLeft(pb.NewDataLiteral<ui16>(0x87F5U), item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x87F5U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x0FEBU);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xFAC3U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xD61FU);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x61FDU);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x87F5U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xF587U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xEB0FU);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xF587U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xC3FAU);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestRotRight) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui8>(0);
+ const auto data1 = pb.NewDataLiteral<ui8>(1);
+ const auto data2 = pb.NewDataLiteral<ui8>(7);
+ const auto data3 = pb.NewDataLiteral<ui8>(10);
+ const auto data4 = pb.NewDataLiteral<ui8>(30);
+ const auto data5 = pb.NewDataLiteral<ui8>(32);
+ const auto data6 = pb.NewDataLiteral<ui8>(40);
+ const auto data7 = pb.NewDataLiteral<ui8>(57);
+ const auto data8 = pb.NewDataLiteral<ui8>(200);
+ const auto data9 = pb.NewDataLiteral<ui8>(255);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui8>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.RotRight(pb.NewDataLiteral<ui16>(0x87F5U), item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x87F5U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xC3FAU);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xEB0FU);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xFD61U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x1FD6U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x87F5U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xF587U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xFAC3U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0xF587U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0x0FEBU);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestConvertIntToBool) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<i64>(0);
+ const auto data1 = pb.NewDataLiteral<i64>(1);
+ const auto data2 = pb.NewDataLiteral<i64>(2);
+ const auto data3 = pb.NewDataLiteral<i64>(1024);
+ const auto data4 = pb.NewDataLiteral<i64>(-1);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i64>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Convert(item, pb.NewDataType(NUdf::TDataType<bool>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.template Get<bool>());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.template Get<bool>());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.template Get<bool>());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.template Get<bool>());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.template Get<bool>());
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFloatAbs) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto data1 = pb.NewDataLiteral<float>(11.433f);
+ const auto data2 = pb.NewDataLiteral<float>(-3.14f);
+ const auto data3 = pb.NewDataLiteral<float>(0.0f);
+ const auto data4 = pb.NewDataLiteral<float>(-HUGE_VALF);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Abs(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 11.433f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 3.14f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.0f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), HUGE_VALF);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIntegerAbs) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto data1 = pb.NewDataLiteral<i32>(1334);
+ const auto data2 = pb.NewDataLiteral<i32>(-4378);
+ const auto data3 = pb.NewDataLiteral<i32>(0);
+ const auto data4 = pb.NewDataLiteral<i32>(std::numeric_limits<i32>::min());
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Abs(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1334);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 4378);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), std::numeric_limits<i32>::min());
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestToIntegral) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto ten = pb.ToIntegral(pb.NewDataLiteral(float(10.0)), pb.NewDataType(NUdf::TDataType<i32>::Id, true));
+ const auto two = pb.ToIntegral(pb.NewDataLiteral(float(2.0)), pb.NewDataType(NUdf::TDataType<ui32>::Id, true));
+ const auto pgmReturn = pb.NewTuple({ten, two});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(0).template Get<i32>(), 10);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(1).template Get<ui32>(), 2U);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFloatToI16) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0f);
+ const auto data1 = pb.NewDataLiteral(0.0f*HUGE_VALF);
+ const auto data2 = pb.NewDataLiteral(-3.14f);
+ const auto data3 = pb.NewDataLiteral(12121324.0f);
+ const auto data4 = pb.NewDataLiteral(-7898.8f);
+ const auto data5 = pb.NewDataLiteral(210000.0f);
+ const auto data6 = pb.NewDataLiteral(HUGE_VALF);
+ const auto data7 = pb.NewDataLiteral(-HUGE_VALF);
+ const auto data8 = pb.NewDataLiteral(-HUGE_VALF);
+ const auto data9 = pb.NewDataLiteral(FLT_MIN/2.0f);
+ const auto type = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i16>::Id, true));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), -3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), -7898);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 0);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoubleToBool) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0);
+ const auto data1 = pb.NewDataLiteral(0.0*HUGE_VAL);
+ const auto data2 = pb.NewDataLiteral(-3.14);
+ const auto data3 = pb.NewDataLiteral(12121324.0);
+ const auto data4 = pb.NewDataLiteral(-7898.8);
+ const auto data5 = pb.NewDataLiteral(210000.0);
+ const auto data6 = pb.NewDataLiteral(HUGE_VAL);
+ const auto data7 = pb.NewDataLiteral(-HUGE_VAL);
+ const auto data8 = pb.NewDataLiteral(-HUGE_VAL);
+ const auto data9 = pb.NewDataLiteral(DBL_MIN/2.0);
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<bool>::Id, true));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoubleToUI32) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0);
+ const auto data1 = pb.NewDataLiteral(0.0*HUGE_VAL);
+ const auto data2 = pb.NewDataLiteral(-3.14);
+ const auto data3 = pb.NewDataLiteral(12121324.0);
+ const auto data4 = pb.NewDataLiteral(-7898.8);
+ const auto data5 = pb.NewDataLiteral(210000.0);
+ const auto data6 = pb.NewDataLiteral(HUGE_VAL);
+ const auto data7 = pb.NewDataLiteral(-HUGE_VAL);
+ const auto data8 = pb.NewDataLiteral(-HUGE_VAL);
+ const auto data9 = pb.NewDataLiteral(DBL_MIN/2.0);
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui32>::Id, true));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 12121324U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 210000U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestUI64ToIntegral) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto data0 = pb.NewDataLiteral<ui64>(0U);
+ const auto data1 = pb.NewDataLiteral<ui64>(1U);
+ const auto data2 = pb.NewDataLiteral<ui64>(std::numeric_limits<i8>::max());
+ const auto data3 = pb.NewDataLiteral<ui64>(std::numeric_limits<ui8>::max());
+ const auto data4 = pb.NewDataLiteral<ui64>(std::numeric_limits<i16>::max());
+ const auto data5 = pb.NewDataLiteral<ui64>(std::numeric_limits<ui16>::max());
+ const auto data6 = pb.NewDataLiteral<ui64>(std::numeric_limits<i32>::max());
+ const auto data7 = pb.NewDataLiteral<ui64>(std::numeric_limits<ui32>::max());
+ const auto data8 = pb.NewDataLiteral<ui64>(std::numeric_limits<i64>::max());
+ const auto data9 = pb.NewDataLiteral<ui64>(std::numeric_limits<ui64>::max());
+
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i8>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui8>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i16>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui16>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i32>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui32>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i64>::Id, true)),
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 0);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 1);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<i8>::max());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), std::numeric_limits<ui8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<ui8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<ui8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<ui8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<ui8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<ui8>::max());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<i16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<i16>::max());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<ui16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<ui16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<ui16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<ui16>::max());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i32>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i32>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<i32>::max());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT(!item.GetElements()[4]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<ui32>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<ui32>::max());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT(!item.GetElements()[4]);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), std::numeric_limits<i64>::max());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT(!item.GetElements()[4]);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT(!item.GetElements()[6]);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestI64ToIntegral) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<i64>::Id);
+ const auto data0 = pb.NewDataLiteral<i64>(0);
+ const auto data1 = pb.NewDataLiteral<i64>(-1);
+ const auto data2 = pb.NewDataLiteral<i64>(std::numeric_limits<i8>::min());
+ const auto data3 = pb.NewDataLiteral<i64>(std::numeric_limits<i8>::max());
+ const auto data4 = pb.NewDataLiteral<i64>(std::numeric_limits<i16>::min());
+ const auto data5 = pb.NewDataLiteral<i64>(std::numeric_limits<i16>::max());
+ const auto data6 = pb.NewDataLiteral<i64>(std::numeric_limits<i32>::min());
+ const auto data7 = pb.NewDataLiteral<i64>(std::numeric_limits<i32>::max());
+ const auto data8 = pb.NewDataLiteral<i64>(std::numeric_limits<i64>::min());
+ const auto data9 = pb.NewDataLiteral<i64>(std::numeric_limits<i64>::max());
+
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i8>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui8>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i16>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui16>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i32>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui32>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui64>::Id, true)),
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<ui64>(), 0);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), -1);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), -1);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -1);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT(!item.GetElements()[6]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), std::numeric_limits<i8>::min());
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i8>::min());
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i8>::min());
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT(!item.GetElements()[6]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i8>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<ui64>(), std::numeric_limits<i8>::max());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i16>::min());
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i16>::min());
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT(!item.GetElements()[6]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), std::numeric_limits<i16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), std::numeric_limits<i16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i16>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<ui64>(), std::numeric_limits<i16>::max());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i32>::min());
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT(!item.GetElements()[6]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), std::numeric_limits<i32>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), std::numeric_limits<i32>::max());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<ui64>(), std::numeric_limits<i32>::max());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT(!item.GetElements()[4]);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT(!item.GetElements()[6]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT(!item.GetElements()[4]);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<ui64>(), std::numeric_limits<i64>::max());
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFloatFromString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("0.0");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("NAN");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-3.14");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("1212.00");
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-7898.8");
+ const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("21E4");
+ const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("+inf");
+ const auto data7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-INF");
+ const auto type = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.StrictFromString(item, pb.NewDataType(NUdf::TDataType<float>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::isnan(item.template Get<float>()));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -3.14f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 1212.f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -7898.8f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 210000.f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), HUGE_VALF);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoubleFromString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("0.0");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("NAN");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-3.14");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("12121324.00");
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-7898.8");
+ const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("21E4");
+ const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("+inf");
+ const auto data7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-INF");
+ const auto type = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.StrictFromString(item, pb.NewDataType(NUdf::TDataType<double>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::isnan(item.template Get<double>()));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -3.14);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 12121324.0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -7898.8);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 210000.0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), HUGE_VAL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -HUGE_VAL);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFloatToString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0f);
+ const auto data1 = pb.NewDataLiteral(0.0f*HUGE_VALF);
+ const auto data2 = pb.NewDataLiteral(-3.14f);
+ const auto data3 = pb.NewDataLiteral(1212.0f);
+ const auto data4 = pb.NewDataLiteral(-7898.8f);
+ const auto data5 = pb.NewDataLiteral(210000.0f);
+ const auto data6 = pb.NewDataLiteral(HUGE_VALF);
+ const auto data7 = pb.NewDataLiteral(-HUGE_VALF);
+ const auto type = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToString(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "0");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "nan");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "-3.14");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "1212");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "-7898.8");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "210000");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "inf");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "-inf");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestInt64ToTimestamp) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(i64(0LL));
+ const auto data1 = pb.NewDataLiteral(i64(1LL));
+ const auto data2 = pb.NewDataLiteral(i64(-1LL));
+ const auto data3 = pb.NewDataLiteral(std::numeric_limits<i64>::min());
+ const auto data4 = pb.NewDataLiteral(std::numeric_limits<i64>::max());
+ const auto data5 = pb.NewDataLiteral(i64(4291747200000000LL));
+ const auto data6 = pb.NewDataLiteral(i64(4291747199999999LL));
+ const auto type = pb.NewDataType(NUdf::TDataType<i64>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id, true));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 0ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 1ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 4291747199999999ULL);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFloatConvertToUint32) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0f);
+ const auto data3 = pb.NewDataLiteral(1212.0f);
+ const auto data5 = pb.NewDataLiteral(210000.0f);
+ const auto type = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(type, {data0, data3, data5});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Convert(item, pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1212U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 210000U);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFloatConvertToInt32) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0f);
+ const auto data2 = pb.NewDataLiteral(-3.14f);
+ const auto data3 = pb.NewDataLiteral(1212.0f);
+ const auto data4 = pb.NewDataLiteral(-7898.8f);
+ const auto data5 = pb.NewDataLiteral(210000.0f);
+ const auto type = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(type, {data0, data2, data3, data4, data5});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Convert(item, pb.NewDataType(NUdf::TDataType<i32>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1212);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -7898);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 210000);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoubleConvertToInt64) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0);
+ const auto data2 = pb.NewDataLiteral(-3.14);
+ const auto data3 = pb.NewDataLiteral(1212.0);
+ const auto data4 = pb.NewDataLiteral(-7898.8);
+ const auto data5 = pb.NewDataLiteral(210000.0);
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, {data0, data2, data3, data4, data5});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Convert(item, pb.NewDataType(NUdf::TDataType<i64>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 0LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -3LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 1212LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -7898LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 210000LL);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoubleToInt64) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0);
+ const auto data2 = pb.NewDataLiteral(-3.14);
+ const auto data3 = pb.NewDataLiteral(1212.0);
+ const auto data4 = pb.NewDataLiteral(-7898.8);
+ const auto data5 = pb.NewDataLiteral(210000.0);
+ const auto data6 = pb.NewDataLiteral(9223372036854776000.0);
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, {data0, data2, data3, data4, data5, data6});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToIntegral(item, pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i64>::Id)));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 0LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -3LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 1212LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -7898LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), 210000LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoubleToString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0);
+ const auto data1 = pb.NewDataLiteral(0.0*HUGE_VAL);
+ const auto data2 = pb.NewDataLiteral(-3.14);
+ const auto data3 = pb.NewDataLiteral(12121324.0);
+ const auto data4 = pb.NewDataLiteral(-7898.8);
+ const auto data5 = pb.NewDataLiteral(210000.0);
+ const auto data6 = pb.NewDataLiteral(HUGE_VAL);
+ const auto data7 = pb.NewDataLiteral(-HUGE_VAL);
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToString(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "0");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "nan");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "-3.14");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "12121324");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "-7898.8");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "210000");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "inf");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "-inf");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestNanvl) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral(0.0f));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral(0.0f*HUGE_VALF));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral(HUGE_VALF));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral(-HUGE_VALF));
+ const auto data4 = pb.NewOptional(pb.NewDataLiteral(FLT_MIN/2.0f));
+ const auto data5 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<float>::Id);
+ const auto data = pb.NewDataLiteral(3.14);
+
+ const auto type = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<float>::Id));
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Nanvl(item, data);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 3.14);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), HUGE_VAL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -HUGE_VAL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), FLT_MIN/2.0f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestConvertToFloat) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto seven = pb.NewDataLiteral(i32(7));
+ const auto pgmReturn = pb.Convert(seven, pb.NewDataType(NUdf::TDataType<float>::Id));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().template Get<float>(), 7.0f);
+ }
+
+ Y_UNIT_TEST_LLVM(TestAppend) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ auto pgmReturn = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ pgmReturn = pb.Append(pgmReturn, pb.NewDataLiteral<ui32>(34));
+ pgmReturn = pb.Append(pgmReturn, pb.NewDataLiteral<ui32>(56));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 2);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestForkedAppend) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {pb.NewDataLiteral<ui32>(34)});
+ const auto list2 = pb.Append(list1, pb.NewDataLiteral<ui32>(56));
+ const auto list3 = pb.Append(list1, pb.NewDataLiteral<ui32>(78));
+ const auto pgmReturn = pb.NewTuple({list2, list3});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator1 = graph->GetValue().GetElement(0).GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator1.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(iterator1.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56);
+ UNIT_ASSERT(!iterator1.Next(item));
+ UNIT_ASSERT(!iterator1.Next(item));
+
+ const auto iterator2 = graph->GetValue().GetElement(1).GetListIterator();
+ UNIT_ASSERT(iterator2.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(iterator2.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 78);
+ UNIT_ASSERT(!iterator2.Next(item));
+ UNIT_ASSERT(!iterator2.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestPrepend) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ auto pgmReturn = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ pgmReturn = pb.Prepend(pb.NewDataLiteral<ui32>(34), pgmReturn);
+ pgmReturn = pb.Prepend(pb.NewDataLiteral<ui32>(56), pgmReturn);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 2);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestPrependOfVoid) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ auto pgmReturn = pb.NewEmptyListOfVoid();
+ pgmReturn = pb.Prepend(pb.NewVoid(), pgmReturn);
+ pgmReturn = pb.Prepend(pb.NewVoid(), pgmReturn);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 0);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ UNIT_ASSERT(!iterator.Skip());
+ UNIT_ASSERT(!iterator.Skip());
+ }
+
+ Y_UNIT_TEST_LLVM(TestForkedPrepend) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto list = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ const auto list1 = pb.Prepend(pb.NewDataLiteral<ui32>(34), list);
+ const auto list2 = pb.Prepend(pb.NewDataLiteral<ui32>(56), list1);
+ const auto list3 = pb.Prepend(pb.NewDataLiteral<ui32>(78), list1);
+ const auto pgmReturn = pb.NewTuple({list2, list3});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator1 = graph->GetValue().GetElement(0).GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator1.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56);
+ UNIT_ASSERT(iterator1.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(!iterator1.Next(item));
+ UNIT_ASSERT(!iterator1.Next(item));
+
+ const auto iterator2 = graph->GetValue().GetElement(1).GetListIterator();
+ UNIT_ASSERT(iterator2.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 78);
+ UNIT_ASSERT(iterator2.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(!iterator2.Next(item));
+ UNIT_ASSERT(!iterator2.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestAppendOfVoid) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ auto pgmReturn = pb.NewEmptyListOfVoid();
+ pgmReturn = pb.Append(pgmReturn, pb.NewVoid());
+ pgmReturn = pb.Append(pgmReturn, pb.NewVoid());
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 0);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ UNIT_ASSERT(!iterator.Skip());
+ UNIT_ASSERT(!iterator.Skip());
+ }
+
+ Y_UNIT_TEST_LLVM(TestExtend) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto emptyList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ const auto list1 = pb.Append(emptyList, pb.NewDataLiteral<ui32>(34));
+ const auto list2 = pb.Append(emptyList, pb.NewDataLiteral<ui32>(56));
+ const auto pgmReturn = pb.Extend(list1, list2);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 2);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestExtend3) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto emptyList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ const auto list1 = pb.Append(emptyList, pb.NewDataLiteral<ui32>(34));
+ const auto list2 = pb.Append(emptyList, pb.NewDataLiteral<ui32>(56));
+ const auto list3 = pb.Append(emptyList, pb.NewDataLiteral<ui32>(7));
+ const auto list4 = pb.Append(list3, pb.NewDataLiteral<ui32>(12));
+ const auto pgmReturn = pb.Extend({ list1, emptyList, list2, list4 });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 4);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 12);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestExtendOverFlows) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0);
+ const auto data1 = pb.NewDataLiteral(1.1);
+ const auto data2 = pb.NewDataLiteral(-3.14);
+ const auto data3 = pb.NewDataLiteral(121324.323);
+ const auto data4 = pb.NewDataLiteral(-7898.8);
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list1 = pb.NewList(type, {data0, data1, data2});
+ const auto list2 = pb.NewList(type, {data3, data4});
+
+ const auto pgmReturn = pb.FromFlow(pb.Extend({pb.ToFlow(list2), pb.ToFlow(list1), pb.ToFlow(list2)}));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 121324.323);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -7898.8);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 1.1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -3.14);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 121324.323);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -7898.8);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestStreamForwardList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto data1 = pb.NewDataLiteral<ui32>(34);
+ const auto data2 = pb.NewDataLiteral<ui32>(56);
+ const auto data3 = pb.NewDataLiteral<ui32>(7);
+ const auto type = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(type, {data1, data2, data3});
+
+ const auto pgmReturn = pb.ForwardList(pb.Iterator(list, {}));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 7);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFlowForwardList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto data1 = pb.NewDataLiteral<ui32>(34);
+ const auto data2 = pb.NewDataLiteral<ui32>(56);
+ const auto data3 = pb.NewDataLiteral<ui32>(7);
+ const auto type = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(type, {data1, data2, data3});
+
+ const auto pgmReturn = pb.ForwardList(pb.ToFlow(list));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 7);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFlowFromOptional) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0);
+ const auto data1 = pb.NewDataLiteral(1.1);
+ const auto data2 = pb.NewDataLiteral(-3.14);
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+
+ const auto opt0 = pb.NewOptional(data0);
+ const auto opt1 = pb.NewOptional(data1);
+ const auto opt2 = pb.NewOptional(data2);
+ const auto opt = pb.NewEmptyOptional(pb.NewOptionalType(type));
+
+
+ const auto pgmReturn = pb.FromFlow(pb.Extend({pb.ToFlow(opt0), pb.ToFlow(opt), pb.ToFlow(opt1), pb.ToFlow(opt), pb.ToFlow(opt2)}));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 1.1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -3.14);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCollectOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0);
+ const auto data1 = pb.NewDataLiteral(1.1);
+ const auto data2 = pb.NewDataLiteral(-3.14);
+ const auto data3 = pb.NewDataLiteral(121324.323);
+ const auto data4 = pb.NewDataLiteral(-7898.8);
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto data = pb.NewList(type, {data0, data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.ToFlow(data));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto list = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0).template Get<double>(), 0.0);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(3).template Get<double>(), 121324.323);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1).template Get<double>(), 1.1);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(2).template Get<double>(), -3.14);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(4).template Get<double>(), -7898.8);
+ }
+
+ Y_UNIT_TEST_LLVM(TestAdd) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto pgmReturn = pb.Add(data1, data2);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue().template Get<ui32>();
+ UNIT_ASSERT_VALUES_EQUAL(res, 3);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSub) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(7);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto pgmReturn = pb.Sub(data1, data2);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue().template Get<ui32>();
+ UNIT_ASSERT_VALUES_EQUAL(res, 5);
+ }
+
+ Y_UNIT_TEST_LLVM(TestMul) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(3);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto pgmReturn = pb.Mul(data1, data2);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue().template Get<ui32>();
+ UNIT_ASSERT_VALUES_EQUAL(res, 6);
+ }
+
+ Y_UNIT_TEST_LLVM(TestDiv) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(17);
+ const auto data2 = pb.NewDataLiteral<ui32>(3);
+ const auto pgmReturn = pb.Div(data1, data2);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT(value);
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 5);
+ }
+
+ Y_UNIT_TEST_LLVM(TestDivZero) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(17);
+ const auto data2 = pb.NewDataLiteral<ui32>(0);
+ const auto pgmReturn = pb.Div(data1, data2);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT(!value);
+ }
+
+ Y_UNIT_TEST_LLVM(TestIDivOverflow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<i32>(Min<i32>());
+ const auto data2 = pb.NewDataLiteral<i32>(-1);
+ const auto pgmReturn = pb.Div(data1, data2);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT(!value);
+ }
+
+ Y_UNIT_TEST_LLVM(TestMod) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(17);
+ const auto data2 = pb.NewDataLiteral<ui32>(3);
+ const auto pgmReturn = pb.Mod(data1, data2);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT(value);
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 2);
+ }
+
+ Y_UNIT_TEST_LLVM(TestModZero) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(17);
+ const auto data2 = pb.NewDataLiteral<ui32>(0);
+ const auto pgmReturn = pb.Mod(data1, data2);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT(!value);
+ }
+
+ Y_UNIT_TEST_LLVM(TestIModOverflow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<i32>(Min<i32>());
+ const auto data2 = pb.NewDataLiteral<i32>(-1);
+ const auto pgmReturn = pb.Mod(data1, data2);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT(!value);
+ }
+
+ Y_UNIT_TEST_LLVM(TestStruct) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto data3 = pb.NewDataLiteral<ui32>(3);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto structObj = pb.NewEmptyStruct();
+ structObj = pb.AddMember(structObj, "x", data2);
+ structObj = pb.AddMember(structObj, "z", data1);
+ structObj = pb.AddMember(structObj, "y", data3);
+ const auto pgmReturn = pb.NewList(dataType, {
+ pb.Member(structObj, "x"),
+ pb.Member(structObj, "y"),
+ pb.Member(structObj, "z")
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMapOverList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<i32>(1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<i32>(2));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<i32>(3));
+ const auto data4 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Sub(pb.Mul(item, data2), data1);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 5);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMapOverStream) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i16>::Id));
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral<i16>(0));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<i16>(-1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<i16>(3));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<i16>(7));
+ const auto data4 = pb.NewOptional(pb.NewDataLiteral<i16>(11));
+ const auto data5 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i16>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5});
+
+ const auto pgmReturn = pb.Map(pb.Iterator(list, {}),
+ [&](TRuntimeNode item) {
+ return pb.Div(data3, item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), -7);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMapOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("PREFIX:");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very large string");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("small");
+ const auto type = pb.NewDataType(NUdf::EDataSlot::String);
+ const auto list = pb.NewList(type, {data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.Map(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return pb.Concat(data0, item);
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "PREFIX:very large string");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "PREFIX:");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "PREFIX:small");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+#ifndef _win_
+ Y_UNIT_TEST_LLVM(TestMapUnwrapThrow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<i8>(1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<i8>(7));
+ const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id);
+ const auto list = pb.LazyList(pb.NewList(dataType, {data1, data0}));
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Sub(data2, pb.Unwrap(item, pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), 6);
+ UNIT_ASSERT_EXCEPTION(iterator.Next(item), yexception);
+ }
+
+ Y_UNIT_TEST_LLVM(TestMapUnwrapThrowMessage) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<i8>(1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<i8>(7));
+ const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id);
+ const auto list = pb.LazyList(pb.NewList(dataType, {data1, data0}));
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Sub(data2, pb.Unwrap(item, pb.Concat(
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("a"),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("b")),
+ "", 0, 0));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), 6);
+ UNIT_ASSERT_EXCEPTION(iterator.Next(item), yexception);
+ }
+
+ Y_UNIT_TEST_LLVM(TestMapEnsureThrowMessage) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i8>::Id);
+ const auto data1 = pb.NewDataLiteral<i8>(1);
+ const auto data2 = pb.NewDataLiteral<i8>(7);
+ const auto list = pb.LazyList(pb.NewList(dataType, {data1, data2}));
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Sub(data2,
+ pb.Ensure(item, pb.Less(item, data2), pb.NewDataLiteral<NUdf::EDataSlot::String>("BAD"),"", 0, 0));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), 6);
+ UNIT_ASSERT_EXCEPTION(iterator.Next(item), yexception);
+ }
+#endif
+ Y_UNIT_TEST_LLVM(TestMapWithCoalesce) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i16>::Id));
+ const auto data0 = pb.NewDataLiteral<i16>(0);
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<i16>(1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<i16>(2));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<i16>(3));
+ const auto data = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i16>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Coalesce(item, data0);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i16>(), 0);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSizeOfOptional) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i16>::Id));
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral<i16>(0));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<i16>(1));
+ const auto data = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i16>::Id);
+ const auto list = pb.NewList(dataType, {data1, data0, data});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Size(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSizeOfOptionalString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<char*>::Id));
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("0123456789"));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("XYZ"));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>(""));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("qwertyuioplkjhgfdsazxcvbnm"));
+ const auto data = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Size(pb.Concat(item, data0));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 20U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 13U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 10U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 36U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMapWithIfExists) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui16>::Id));
+ const auto data0 = pb.NewDataLiteral<ui16>(666);
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<ui16>(1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<ui16>(2));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<ui16>(3));
+ const auto data = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui16>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.If(pb.Exists(item), pb.Increment(pb.Unwrap(item, pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)), data0);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 666);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMapOverListLazy) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1U);
+ const auto data2 = pb.NewDataLiteral<ui32>(2U);
+ const auto data3 = pb.NewDataLiteral<ui32>(3U);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.Map(pb.LazyList(list),
+ [&](TRuntimeNode item) {
+ return pb.Add(pb.Mul(item, data2), data1);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 5);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 7);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMapOverOptional) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto list = pb.NewOptional(data1);
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Add(pb.Mul(item, data2), data1);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT(value);
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 3);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFloatMinMax) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<float>(-NAN);
+ const auto data2 = pb.NewDataLiteral<float>(HUGE_VALF);
+ const auto data3 = pb.NewDataLiteral<float>(3.14f);
+ const auto data4 = pb.NewDataLiteral<float>(-2.13f);
+ const auto data5 = pb.NewDataLiteral<float>(-HUGE_VALF);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5});
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode left) {
+ return pb.Map(list,
+ [&](TRuntimeNode right) {
+ return pb.NewTuple({pb.Min(left, right), pb.Max(left, right)});
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::isnan(item.GetElement(0).template Get<float>()));
+ UNIT_ASSERT(std::isnan(item.GetElement(1).template Get<float>()));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), 3.14f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -2.13f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), 3.14f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), 3.14f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), 3.14f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), 3.14f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -2.13f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -2.13f);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -2.13f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -2.13f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), HUGE_VALF);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -2.13f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<float>(), -HUGE_VALF);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<float>(), -HUGE_VALF);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFloatMod) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<float>(-1.75f);
+ const auto data2 = pb.NewDataLiteral<float>(3.14f);
+ const auto data3 = pb.NewDataLiteral<float>(-6.28f);
+ const auto data4 = pb.NewDataLiteral<float>(7.28f);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Mod(item, data2);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -1.75f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.0f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.0f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 1.0f);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoubleMod) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<double>(-1.75);
+ const auto data2 = pb.NewDataLiteral<double>(3.14);
+ const auto data3 = pb.NewDataLiteral<double>(-6.28);
+ const auto data4 = pb.NewDataLiteral<double>(7.28);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Mod(item, data2);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -1.75);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 1.0);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDiscardOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.Discard(pb.ToFlow(list)));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestHead) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto itemType = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto listType = pb.NewListType(itemType);
+
+ const auto data0 = pb.NewEmptyList(itemType);
+ const auto data1 = pb.NewList(itemType, {pb.NewDataLiteral<float>(-1.5f), pb.NewDataLiteral<float>(0.f), pb.NewDataLiteral<float>(3.14f)});
+ const auto data2 = pb.NewList(itemType, {pb.NewDataLiteral<float>(3.14f), pb.NewDataLiteral<float>(-1.5f), pb.NewDataLiteral<float>(0.f)});
+ const auto data3 = pb.LazyList(pb.NewList(itemType, {pb.NewDataLiteral<float>(1.1f), pb.NewDataLiteral<float>(-2.2f)}));
+
+ const auto list = pb.NewList(listType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.Map(list, [&](TRuntimeNode item) { return pb.Head(item); });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -1.5f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 1.1f);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestLast) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto itemType = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto listType = pb.NewListType(itemType);
+
+ const auto data0 = pb.NewEmptyList(itemType);
+ const auto data1 = pb.NewList(itemType, {pb.NewDataLiteral<float>(-1.5f), pb.NewDataLiteral<float>(0.f), pb.NewDataLiteral<float>(3.14f)});
+ const auto data2 = pb.NewList(itemType, {pb.NewDataLiteral<float>(3.14f), pb.NewDataLiteral<float>(-1.5f), pb.NewDataLiteral<float>(0.f)});
+ const auto data3 = pb.LazyList(pb.NewList(itemType, {pb.NewDataLiteral<float>(1.1f), pb.NewDataLiteral<float>(-2.2f)}));
+
+ const auto list = pb.NewList(listType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.Map(list, [&](TRuntimeNode item) { return pb.Last(item); });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 3.14f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.f);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -2.2f);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCoalesce) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto data3 = pb.NewDataLiteral<ui32>(3);
+ auto pgmReturn = pb.NewEmptyStruct();
+ pgmReturn = pb.AddMember(pgmReturn, "A", pb.Coalesce(
+ pb.NewOptional(data1), data2));
+ pgmReturn = pb.AddMember(pgmReturn, "B", pb.Coalesce(
+ pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id), data2));
+ pgmReturn = pb.AddMember(pgmReturn, "C", pb.Coalesce(
+ pb.NewOptional(data2), pb.NewOptional(data3)));
+ pgmReturn = pb.AddMember(pgmReturn, "D", pb.Coalesce(
+ pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id), pb.NewOptional(data3)));
+ pgmReturn = pb.AddMember(pgmReturn, "E", pb.Coalesce(
+ pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id), pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id)));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0).template Get<ui32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1).template Get<ui32>(), 2);
+ UNIT_ASSERT(value.GetElement(2));
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2).template Get<ui32>(), 2);
+ UNIT_ASSERT(value.GetElement(3));
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(3).template Get<ui32>(), 3);
+ UNIT_ASSERT(!value.GetElement(4));
+ }
+
+ Y_UNIT_TEST_LLVM(TestExists) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui64>(1);
+ const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui64>::Id));
+ const auto list = pb.NewList(optionalType, {pb.NewOptional(data1), pb.NewEmptyOptional(optionalType)});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Exists(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIf) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto truth = pb.NewDataLiteral(true);
+ const auto falsehood = pb.NewDataLiteral(false);
+ const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<bool>::Id), {truth, falsehood});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.If(item, data1, data2);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIfPresent) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto filled = pb.NewOptional(data1);
+ const auto empty = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id)), {filled, empty});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.IfPresent({item},
+ [&](TRuntimeNode::TList unpacked){
+ return unpacked.front();
+ }, data2);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIfPresentTwo) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<i8>(+1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<i8>(-1));
+ const auto empty1 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id);
+
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<bool>(true));
+ const auto data4= pb.NewOptional(pb.NewDataLiteral<bool>(false));
+ const auto empty2 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id);
+
+ const auto list1 = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id)), {data1, empty1, data2});
+ const auto list2 = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id)), {data3, empty2, data4});
+
+ const auto pgmReturn = pb.FlatMap(list1,
+ [&](TRuntimeNode item1) {
+ return pb.FlatMap(list2,
+ [&](TRuntimeNode item2) {
+ return pb.IfPresent({item1, item2},
+ [&](TRuntimeNode::TList unpacked) {
+ return pb.NewOptional(pb.If(unpacked.back(), pb.Minus(unpacked.front()), unpacked.front()));
+ }, empty1);
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -1);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIfPresentThree) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<i8>(+1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<i8>(-1));
+ const auto empty1 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id);
+
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<bool>(true));
+ const auto data4= pb.NewOptional(pb.NewDataLiteral<bool>(false));
+ const auto empty2 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id);
+
+ const auto data5 = pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i8>(5)));
+ const auto data6 = pb.NewOptional(pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id));
+ const auto type2 = pb.NewOptionalType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id)));
+ const auto empty3 = pb.NewEmptyOptional(type2);
+
+ const auto list1 = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id)), {data1, empty1, data2});
+ const auto list2 = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id)), {data3, empty2, data4});
+ const auto list3 = pb.NewList(type2, {data5, empty3, data6});
+
+ const auto pgmReturn = pb.FlatMap(list1,
+ [&](TRuntimeNode item1) {
+ return pb.FlatMap(list2,
+ [&](TRuntimeNode item2) {
+ return pb.FlatMap(list3,
+ [&](TRuntimeNode item3) {
+ return pb.IfPresent({item1, item2, item3},
+ [&](TRuntimeNode::TList unpacked) {
+ return pb.NewOptional(pb.If(unpacked[1],
+ pb.Add(unpacked.front(), pb.Coalesce(unpacked.back(), pb.NewDataLiteral<i8>(3))),
+ pb.Sub(unpacked.front(), pb.Coalesce(unpacked.back(), pb.NewDataLiteral<i8>(7)))
+ ));
+ }, empty1);
+ });
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +6);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -6);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), +2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -6);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -8);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIfPresentSame) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<i8>(+1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<i8>(-1));
+ const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i8>::Id);
+
+ const auto list = pb.NewList(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i8>::Id)), {data1, data0, data2});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ const auto minus = pb.Minus(item);
+ return pb.NewTuple({
+ pb.IfPresent({minus},
+ [&](TRuntimeNode::TList unpacked) {
+ return pb.Abs(unpacked.front());
+ }, data0),
+ pb.IfPresent({minus},
+ [&](TRuntimeNode::TList unpacked) {
+ return pb.Minus(unpacked.front());
+ }, data0),
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i8>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i8>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i8>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i8>(), -1);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIncDec) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(7);
+ auto pgmReturn = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ pgmReturn = pb.Append(pgmReturn, pb.Increment(data1));
+ pgmReturn = pb.Append(pgmReturn, pb.Decrement(data1));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 8);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 6);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestLogical) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto truth = pb.NewDataLiteral(true);
+ const auto falsehood = pb.NewDataLiteral(false);
+ auto pgmReturn = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<bool>::Id));
+
+ pgmReturn = pb.Append(pgmReturn, pb.And({truth, truth}));
+ pgmReturn = pb.Append(pgmReturn, pb.And({truth, falsehood}));
+ pgmReturn = pb.Append(pgmReturn, pb.And({falsehood, truth}));
+ pgmReturn = pb.Append(pgmReturn, pb.And({falsehood, falsehood}));
+
+ pgmReturn = pb.Append(pgmReturn, pb.Or({truth, truth}));
+ pgmReturn = pb.Append(pgmReturn, pb.Or({falsehood, truth}));
+ pgmReturn = pb.Append(pgmReturn, pb.Or({truth, falsehood}));
+ pgmReturn = pb.Append(pgmReturn, pb.Or({falsehood, falsehood}));
+
+ pgmReturn = pb.Append(pgmReturn, pb.Xor({truth, truth}));
+ pgmReturn = pb.Append(pgmReturn, pb.Xor({falsehood, truth}));
+ pgmReturn = pb.Append(pgmReturn, pb.Xor({truth, falsehood}));
+ pgmReturn = pb.Append(pgmReturn, pb.Xor({falsehood, falsehood}));
+
+ pgmReturn = pb.Append(pgmReturn, pb.Not(truth));
+ pgmReturn = pb.Append(pgmReturn, pb.Not(falsehood));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // not
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestZip) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {
+ pb.NewDataLiteral<ui32>(34),
+ pb.NewDataLiteral<ui32>(56)
+ });
+ const auto list2 = pb.NewList(pb.NewDataType(NUdf::TDataType<char*>::Id), {
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("Q"),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("E"),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("W")
+ });
+
+ const auto pgmReturn = pb.Zip({list1, list2});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto list = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 2U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0).GetElement(0).template Get<ui32>(), 34);
+ UNBOXED_VALUE_STR_EQUAL(list.GetElement(0).GetElement(1), "Q");
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1).GetElement(0).template Get<ui32>(), 56);
+ UNBOXED_VALUE_STR_EQUAL(list.GetElement(1).GetElement(1), "E");
+ }
+
+ Y_UNIT_TEST_LLVM(TestZipLazy) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {
+ pb.NewDataLiteral<ui32>(34),
+ pb.NewDataLiteral<ui32>(56)
+ });
+ const auto list2 = pb.NewList(pb.NewDataType(NUdf::TDataType<char*>::Id), {
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("Q"),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("E"),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("W")
+ });
+
+ const auto pgmReturn = pb.Zip({pb.LazyList(list1), list2});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 34);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "Q");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 56);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "E");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestZipAll) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {
+ pb.NewDataLiteral<ui32>(34),
+ pb.NewDataLiteral<ui32>(56)
+ });
+ const auto list2 = pb.NewList(pb.NewDataType(NUdf::TDataType<char*>::Id), {
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("Q"),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("E"),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("W")
+ });
+
+ const auto pgmReturn = pb.ZipAll({list1, list2});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto list = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 3U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0).GetElement(0).template Get<ui32>(), 34);
+ UNBOXED_VALUE_STR_EQUAL(list.GetElement(0).GetElement(1), "Q");
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1).GetElement(0).template Get<ui32>(), 56);
+ UNBOXED_VALUE_STR_EQUAL(list.GetElement(1).GetElement(1), "E");
+ UNIT_ASSERT(!list.GetElement(2).GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(list.GetElement(2).GetElement(1), "W");
+ }
+
+ Y_UNIT_TEST_LLVM(TestZipAllLazy) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {
+ pb.NewDataLiteral<ui32>(34),
+ pb.NewDataLiteral<ui32>(56)
+ });
+ const auto list2 = pb.NewList(pb.NewDataType(NUdf::TDataType<char*>::Id), {
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("Q"),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("E"),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("W")
+ });
+
+ const auto pgmReturn = pb.ZipAll({list1, pb.LazyList(list2)});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 34);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "Q");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 56);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "E");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "W");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestReduce) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<ui32>(3));
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ const auto list = pb.NewList(dataType, {data1, data2, data3});
+ const auto empty = pb.AddMember(pb.AddMember(
+ pb.NewEmptyStruct(), "Min", pb.NewEmptyOptional(dataType)),
+ "Max", pb.NewEmptyOptional(dataType));
+ const auto pgmReturn = pb.Reduce(list, empty,
+ [&](TRuntimeNode item, TRuntimeNode state1) {
+ return pb.AddMember(
+ pb.AddMember(pb.NewEmptyStruct(), "Min", pb.AggrMin(pb.Member(state1, "Min"), item)),
+ "Max", pb.AggrMax(pb.Member(state1, "Max"), item)
+ );
+ },
+ [&](TRuntimeNode state) {
+ return state;
+ }, empty,
+ [&](TRuntimeNode state1, TRuntimeNode state2) {
+ return pb.AddMember(
+ pb.AddMember(pb.NewEmptyStruct(), "Min", pb.AggrMin(pb.Member(state1, "Min"), pb.Member(state2, "Min")))
+ , "Max", pb.AggrMax(pb.Member(state1, "Max"), pb.Member(state2, "Max"))
+ );
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1).template Get<ui32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0).template Get<ui32>(), 3);
+ }
+
+ Y_UNIT_TEST_LLVM(TestReduceOverStream) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<ui32>(3));
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ const auto list = pb.NewList(dataType, {data1, data2, data3});
+ const auto empty = pb.AddMember(pb.AddMember(
+ pb.NewEmptyStruct(), "Min", pb.NewEmptyOptional(dataType)),
+ "Max", pb.NewEmptyOptional(dataType));
+ const auto pgmReturn = pb.Reduce(pb.Iterator(list, {}), empty,
+ [&](TRuntimeNode item, TRuntimeNode state1) {
+ return pb.AddMember(
+ pb.AddMember(pb.NewEmptyStruct(), "Min", pb.AggrMin(pb.Member(state1, "Min"), item)),
+ "Max", pb.AggrMax(pb.Member(state1, "Max"), item)
+ );
+ },
+ [&](TRuntimeNode state) {
+ return state;
+ }, empty,
+ [&](TRuntimeNode state1, TRuntimeNode state2) {
+ return pb.AddMember(
+ pb.AddMember(pb.NewEmptyStruct(), "Min", pb.AggrMin(pb.Member(state1, "Min"), pb.Member(state2, "Min")))
+ , "Max", pb.AggrMax(pb.Member(state1, "Max"), pb.Member(state2, "Max"))
+ );
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1).template Get<ui32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0).template Get<ui32>(), 3);
+ }
+
+ Y_UNIT_TEST_LLVM(TestListLength) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)});
+ const auto pgmReturn = pb.Length(list);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().template Get<ui64>(), 2);
+ }
+
+ Y_UNIT_TEST_LLVM(TestReverse) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)});
+ const auto pgmReturn = pb.Reverse(list);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 56);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipForAppend) {
+ const ui32 n = 100;
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ for (ui32 i = 0; i < n; ++i) {
+ auto list = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ for (ui32 j = 0; j < n; ++j) {
+ list = pb.Append(list, pb.NewDataLiteral(j));
+ }
+
+ const auto skippedList = pb.Skip(list, pb.NewDataLiteral<ui64>(i));
+ auto changedList = skippedList;
+ for (ui32 j = 0; j < n; ++j) {
+ changedList = pb.Prepend(pb.NewDataLiteral(n + n - 1 - j), changedList);
+ }
+
+ auto pgmReturn = pb.NewEmptyList(list.GetStaticType());
+ pgmReturn = pb.Append(pgmReturn, list);
+ pgmReturn = pb.Append(pgmReturn, skippedList);
+ pgmReturn = pb.Append(pgmReturn, changedList);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iteratorLists = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue test, item;
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ auto iterator = test.GetListIterator();
+ for (ui32 j = 0; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ iterator = test.GetListIterator();
+ for (ui32 j = i; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ iterator = test.GetListIterator();
+ for (ui32 j = 0; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j + n);
+ }
+
+ for (ui32 j = i; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iteratorLists.Next(test));
+ UNIT_ASSERT(!iteratorLists.Next(test));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipForPrepend) {
+ const ui32 n = 100;
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ for (ui32 i = 0; i < n; ++i) {
+ auto list = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ for (ui32 j = 0; j < n; ++j) {
+ list = pb.Prepend(pb.NewDataLiteral(n - 1 - j), list);
+ }
+
+ const auto skippedList = pb.Skip(list, pb.NewDataLiteral<ui64>(i));
+ auto changedList = skippedList;
+ for (ui32 j = 0; j < n; ++j) {
+ changedList = pb.Prepend(pb.NewDataLiteral(n + n - 1 - j), changedList);
+ }
+
+ const auto pgmReturn = pb.NewList(list.GetStaticType(), {list, skippedList, changedList});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iteratorLists = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue test, item;
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ auto iterator = test.GetListIterator();
+ for (ui32 j = 0; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ iterator = test.GetListIterator();
+ for (ui32 j = i; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ iterator = test.GetListIterator();
+ for (ui32 j = 0; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j + n);
+ }
+
+ for (ui32 j = i; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iteratorLists.Next(test));
+ UNIT_ASSERT(!iteratorLists.Next(test));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestTakeForAppend) {
+ const ui32 n = 100;
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ for (ui32 i = 0; i < n; ++i) {
+ auto list = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ for (ui32 j = 0; j < n; ++j) {
+ list = pb.Append(list, pb.NewDataLiteral(j));
+ }
+
+ const auto skippedList = pb.Take(list, pb.NewDataLiteral<ui64>(i));
+ auto changedList = skippedList;
+ for (ui32 j = 0; j < n; ++j) {
+ changedList = pb.Append(changedList, pb.NewDataLiteral(n + j));
+ }
+
+ const auto pgmReturn = pb.NewList(list.GetStaticType(), {list, skippedList, changedList});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iteratorLists = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue test, item;
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ auto iterator = test.GetListIterator();
+ for (ui32 j = 0; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ iterator = test.GetListIterator();
+ for (ui32 j = 0; j < i; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ iterator = test.GetListIterator();
+ for (ui32 j = 0; j < i; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ for (ui32 j = 0; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j + n);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iteratorLists.Next(test));
+ UNIT_ASSERT(!iteratorLists.Next(test));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestTakeForPrepend) {
+ const ui32 n = 100;
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ for (ui32 i = 0; i < n; ++i) {
+ auto list = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ for (ui32 j = 0; j < n; ++j) {
+ list = pb.Prepend(pb.NewDataLiteral(n - 1- j), list);
+ }
+
+ const auto skippedList = pb.Take(list, pb.NewDataLiteral<ui64>(i));
+ auto changedList = skippedList;
+ for (ui32 j = 0; j < n; ++j) {
+ changedList = pb.Append(changedList, pb.NewDataLiteral(n + j));
+ }
+
+ const auto pgmReturn = pb.NewList(list.GetStaticType(), {list, skippedList, changedList});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iteratorLists = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue test, item;
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ auto iterator = test.GetListIterator();
+ for (ui32 j = 0; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ iterator = test.GetListIterator();
+ for (ui32 j = 0; j < i; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+
+ UNIT_ASSERT(iteratorLists.Next(test));
+ iterator = test.GetListIterator();
+ for (ui32 j = 0; j < i; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j);
+ }
+
+ for (ui32 j = 0; j < n; ++j) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), j + n);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iteratorLists.Next(test));
+ UNIT_ASSERT(!iteratorLists.Next(test));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestReplicate) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto pgmReturn = pb.Replicate(pb.NewDataLiteral<ui32>(34),
+ pb.NewDataLiteral<ui64>(4), "", 0, 0);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ for (ui32 i = 0; i < 4; ++i) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIntegralCasts) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(258);
+ const auto pgmReturn = pb.Convert(data1, pb.NewDataType(NUdf::TDataType<ui8>::Id));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue().template Get<ui8>();
+ UNIT_ASSERT_VALUES_EQUAL(res, 2);
+ }
+
+ Y_UNIT_TEST_LLVM(TestCastFourIntoTrue) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(4);
+ const auto pgmReturn = pb.Convert(data1, pb.NewDataType(NUdf::TDataType<bool>::Id));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue().template Get<bool>();
+ UNIT_ASSERT(res);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSubstring) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("aabbccc");
+ const auto start1 = pb.NewDataLiteral<ui32>(3);
+ const auto count1 = pb.NewDataLiteral<ui32>(2);
+ const auto start2 = pb.NewDataLiteral<ui32>(4);
+ const auto count2 = pb.NewDataLiteral<ui32>(10);
+ const auto start3 = pb.NewDataLiteral<ui32>(10);
+ const auto count3 = pb.NewDataLiteral<ui32>(1);
+ const auto start4 = pb.NewDataLiteral<ui32>(0);
+ const auto count4 = pb.NewDataLiteral<ui32>(3);
+
+ const auto dataType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<ui32>::Id)});
+ const auto list = pb.NewList(dataType, {
+ pb.NewTuple({start1, count1}),
+ pb.NewTuple({start2, count2}),
+ pb.NewTuple({start3, count3}),
+ pb.NewTuple({start4, count4})
+ });
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Substring(data1, pb.Nth(item, 0), pb.Nth(item, 1));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "bc");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "ccc");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "aab");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSubstringOptionalArgs) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto null = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id);
+ const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<const char*>::Id);
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>(""));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("0123456789"));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("abcdefghijklmnopqrstuvwxyz"));
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<const char*>::Id));
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.Substring(item, null, null),
+ pb.Substring(item, pb.NewOptional(pb.NewDataLiteral<ui32>(7U)), null),
+ pb.Substring(item, null, pb.NewOptional(pb.NewDataLiteral<ui32>(6U))),
+ pb.Substring(item, pb.NewOptional(pb.NewDataLiteral<ui32>(3U)), pb.NewOptional(pb.NewDataLiteral<ui32>(17U)))
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0U));
+ UNIT_ASSERT(!item.GetElement(1U));
+ UNIT_ASSERT(!item.GetElement(2U));
+ UNIT_ASSERT(!item.GetElement(3U));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3U), "");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "0123456789");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "789");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "012345");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3U), "3456789");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "abcdefghijklmnopqrstuvwxyz");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "hijklmnopqrstuvwxyz");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "abcdef");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(3U), "defghijklmnopqrst");
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFindAndSubstring) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<const char*>::Id));
+ const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<const char*>::Id);
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>(""));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("_/>_/>aab*SEP*bccc"));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("*SEP*_/>a>><<___*SEP*a_/>b54b*SEP*c_/>3434cc*SEP*"));
+ const auto data4 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("(none)"));
+
+ const auto sep1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("");
+ const auto sep2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("_/>");
+ const auto sep3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("*SEP*");
+
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4});
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<const char*>::Id), {sep1, sep2, sep3});
+ const auto null = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<ui32>::Id);
+
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode data) {
+ return pb.Map(list1,
+ [&](TRuntimeNode sep) {
+ const auto first = pb.Find(data, sep, null);
+ const auto last = pb.RFind(data, sep, null);
+ const auto len = pb.Size(sep);
+ return pb.NewTuple({
+ pb.Substring(data, null, first),
+ pb.Substring(data, pb.Add(first, len), pb.If(pb.Less(first, last), pb.Sub(last, pb.Add(first, len)), pb.NewDataLiteral<ui32>(0U))),
+ pb.Substring(data, pb.Add(last, len), null)
+ });
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0U));
+ UNIT_ASSERT(!item.GetElement(1U));
+ UNIT_ASSERT(!item.GetElement(2U));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0U));
+ UNIT_ASSERT(!item.GetElement(1U));
+ UNIT_ASSERT(!item.GetElement(2U));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0U));
+ UNIT_ASSERT(!item.GetElement(1U));
+ UNIT_ASSERT(!item.GetElement(2U));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "_/>_/>aab*SEP*bccc");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "aab*SEP*bccc");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "_/>_/>aab");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "bccc");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "*SEP*_/>a>><<___*SEP*a_/>b54b*SEP*c_/>3434cc*SEP*");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "*SEP*");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "a>><<___*SEP*a_/>b54b*SEP*c");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "3434cc*SEP*");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "_/>a>><<___*SEP*a_/>b54b*SEP*c_/>3434cc");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "(none)");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "(none)");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "(none)");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "(none)");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0U), "(none)");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1U), "(none)");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(2U), "(none)");
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFindFromPos) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<NUdf::EDataSlot::String>("_</|>0123456789</|></|>abcxyz</|>*");
+ const auto sep = pb.NewDataLiteral<NUdf::EDataSlot::String>("</|>");
+
+ const auto list = pb.ListFromRange(pb.NewDataLiteral<ui32>(0U), pb.Size(data), pb.NewDataLiteral<ui32>(2U));
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Find(data, sep, item), pb.RFind(data, sep, item)});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 1U);
+ UNIT_ASSERT(!item.GetElement(1U));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 15U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 1U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 19U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 15U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 19U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 15U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 29U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 19U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 29U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 19U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 29U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 19U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 29U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 19U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui32>(), 29U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 19U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0U));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 29U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0U));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui32>(), 29U);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestListFromRange) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui16>(0);
+ const auto data1 = pb.NewDataLiteral<ui16>(9);
+ const auto data2 = pb.NewDataLiteral<ui16>(1);
+ const auto data3 = pb.NewDataLiteral<ui16>(2);
+ const auto data4 = pb.NewDataLiteral<ui16>(3);
+
+ const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui16>::Id), {data2, data3, data4});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ const auto l = pb.ListFromRange(data0, data1, item);
+ return pb.NewTuple({pb.HasItems(l), pb.Length(l), pb.Head(l), pb.Last(l)});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0U).template Get<bool>());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui64>(), 9ULL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2U).template Get<ui16>(), 0U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(3U).template Get<ui16>(), 8U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0U).template Get<bool>());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui64>(), 5ULL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2U).template Get<ui16>(), 0U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(3U).template Get<ui16>(), 8U);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0U).template Get<bool>());
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<ui64>(), 3ULL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2U).template Get<ui16>(), 0U);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(3U).template Get<ui16>(), 6U);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSize) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("aaa");
+ const auto data2 = pb.NewDataLiteral<ui32>(3);
+ const auto data3 = pb.NewDataLiteral<ui64>(7878786987536ull);
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("qqqqq");
+ const auto pgmReturn = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ {pb.Size(data1), pb.Size(data2), pb.Size(data3), pb.Size(data4)});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 8);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 5);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestEnumerate) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)});
+ const auto pgmReturn = pb.Enumerate(list1);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 34);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 56);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestEnumerateLazy) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)});
+ const auto pgmReturn = pb.Enumerate(pb.LazyList(list1));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 34);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 56);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestEnumerateLazyThenReverse) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)});
+ const auto pgmReturn = pb.Reverse(pb.Enumerate(pb.LazyList(list1)));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 56);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 34);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestEnumerateLazyThenSkip) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)});
+ const auto one = pb.NewDataLiteral<ui64>(1);
+ const auto list = pb.Enumerate(pb.LazyList(list1));
+ const auto skip = pb.Skip(list, one);
+ const auto pgmReturn = pb.NewTuple({skip, pb.Length(list), pb.Length(skip)});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetElement(0).GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 56);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(1).template Get<ui64>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(2).template Get<ui64>(), 1);
+ }
+
+ Y_UNIT_TEST_LLVM(TestEnumerateLazyThenTake) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list1 = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ {pb.NewDataLiteral<ui32>(34), pb.NewDataLiteral<ui32>(56)});
+ const auto one = pb.NewDataLiteral<ui64>(1);
+ const auto list = pb.Enumerate(pb.LazyList(list1));
+ const auto take = pb.Take(list, one);
+ const auto pgmReturn = pb.NewTuple({take, pb.Length(list), pb.Length(take)});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetElement(0).GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 34);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(1).template Get<ui64>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(2).template Get<ui64>(), 1);
+ }
+
+ template<bool LLVM>
+ void TestSortImpl(bool asc) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(3);
+
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("aaa");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("qqq");
+
+ const auto keyType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto payloadType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ auto structType = pb.NewEmptyStructType();
+ structType = pb.NewStructType(structType, "payload", payloadType);
+ structType = pb.NewStructType(structType, "key", keyType);
+
+ std::vector<std::pair<std::string_view, TRuntimeNode>> map1 = {
+ { "key", key1 },
+ { "payload", payload1 }
+ };
+
+ std::vector<std::pair<std::string_view, TRuntimeNode>> map2 = {
+ { "key", key2 },
+ { "payload", payload2 }
+ };
+
+ std::vector<std::pair<std::string_view, TRuntimeNode>> map3 = {
+ { "key", key3 },
+ { "payload", payload3 }
+ };
+
+ const auto list = pb.NewList(structType, {
+ pb.NewStruct(map2),
+ pb.NewStruct(map1),
+ pb.NewStruct(map3)
+ });
+
+ {
+ const auto pgmReturn = pb.Sort(list, pb.NewDataLiteral(asc),
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "key");
+ });
+
+ if (asc) {
+ // ascending sort
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 1);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "aaa");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 2);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 3);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "qqq");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ } else {
+ // descending sort
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 3);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "qqq");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 2);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui32>(), 1);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "aaa");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestSort) {
+ TestSortImpl<LLVM>(true);
+ TestSortImpl<LLVM>(false);
+ }
+
+ using TTriple = std::tuple<ui32, ui32, ui32>;
+
+ TRuntimeNode TupleOrder(TProgramBuilder& pb, bool asc1, bool asc2, bool asc3) {
+ TVector<TRuntimeNode> ascending(3);
+ ascending[0] = pb.NewDataLiteral(asc1);
+ ascending[1] = pb.NewDataLiteral(asc2);
+ ascending[2] = pb.NewDataLiteral(asc3);
+
+ TVector<TType*> tupleTypes(3);
+ tupleTypes[0] = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ tupleTypes[1] = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ tupleTypes[2] = pb.NewDataType(NUdf::TDataType<bool>::Id);
+
+ return pb.NewTuple(pb.NewTupleType(tupleTypes), ascending);
+ }
+
+ template<bool LLVM>
+ TVector<TTriple> SortTuples(TSetup<LLVM>& setup, TRuntimeNode list, TRuntimeNode order)
+ {
+ auto& pb = *setup.PgmBuilder;
+ const auto pgmReturn = pb.Sort(list, order, [](TRuntimeNode item) { return item; });
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ TVector<TTriple> result;
+ for (NUdf::TUnboxedValue value; iterator.Next(value);) {
+ ui32 first = value.GetElement(0).template Get<ui32>();
+ ui32 second = value.GetElement(1).template Get<ui32>();
+ ui32 third = value.GetElement(2).template Get<ui32>();
+ result.push_back(TTriple{ first, second, third });
+ }
+ UNIT_ASSERT(!iterator.Skip());
+ return result;
+ }
+
+ Y_UNIT_TEST_LLVM(TestSortTuples) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listMaker = [&]()
+ {
+ TTriple testData[] = {
+ { 1, 1, 1 },
+ { 1, 1, 2 },
+ { 1, 2, 3 },
+ { 1, 3, 0 },
+ { 2, 0, 1 },
+ { 2, 1, 0 },
+ };
+
+ TVector<TRuntimeNode> tuplesList;
+ for (ui32 i = 0; i < Y_ARRAY_SIZE(testData); i++) {
+ TVector<TRuntimeNode> elements(3);
+ elements[0] = pb.NewDataLiteral(std::get<0>(testData[i]));
+ elements[1] = pb.NewDataLiteral(std::get<1>(testData[i]));
+ elements[2] = pb.NewDataLiteral(std::get<2>(testData[i]));
+ tuplesList.push_back(pb.NewTuple(elements));
+ }
+
+ TVector<TType*> tupleTypes(3);
+ tupleTypes[0] = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ tupleTypes[1] = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ tupleTypes[2] = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+
+ return pb.NewList(pb.NewTupleType(tupleTypes), tuplesList);
+ };
+
+ {
+ TRuntimeNode order = TupleOrder(pb, true, true, true);
+ TTriple expectedData[] = {
+ { 1, 1, 1 },
+ { 1, 1, 2 },
+ { 1, 2, 3 },
+ { 1, 3, 0 },
+ { 2, 0, 1 },
+ { 2, 1, 0 },
+ };
+ TVector<TTriple> expected(expectedData, expectedData + sizeof(expectedData) / sizeof(*expectedData));
+ TVector<TTriple> result = SortTuples<LLVM>(setup, listMaker(), order);
+ UNIT_ASSERT_EQUAL(result, expected);
+ }
+
+ {
+ TRuntimeNode order = TupleOrder(pb, false, false, false);
+ TTriple expectedData[] = {
+ { 2, 1, 0 },
+ { 2, 0, 1 },
+ { 1, 3, 0 },
+ { 1, 2, 3 },
+ { 1, 1, 2 },
+ { 1, 1, 1 },
+ };
+ TVector<TTriple> expected(expectedData, expectedData + sizeof(expectedData) / sizeof(*expectedData));
+ TVector<TTriple> result = SortTuples<LLVM>(setup, listMaker(), order);
+ UNIT_ASSERT_EQUAL(result, expected);
+ }
+
+ {
+ TRuntimeNode order = TupleOrder(pb, true, false, true);
+ TTriple expectedData[] = {
+ { 1, 3, 0 },
+ { 1, 2, 3 },
+ { 1, 1, 1 },
+ { 1, 1, 2 },
+ { 2, 1, 0 },
+ { 2, 0, 1 },
+ };
+ TVector<TTriple> expected(expectedData, expectedData + sizeof(expectedData) / sizeof(*expectedData));
+ TVector<TTriple> result = SortTuples<LLVM>(setup, listMaker(), order);
+ UNIT_ASSERT_EQUAL(result, expected);
+ }
+
+ {
+ TRuntimeNode order = TupleOrder(pb, false, true, false);
+ TTriple expectedData[] = {
+ { 2, 0, 1 },
+ { 2, 1, 0 },
+ { 1, 1, 2 },
+ { 1, 1, 1 },
+ { 1, 2, 3 },
+ { 1, 3, 0 },
+ };
+ TVector<TTriple> expected(expectedData, expectedData + sizeof(expectedData) / sizeof(*expectedData));
+ TVector<TTriple> result = SortTuples<LLVM>(setup, listMaker(), order);
+ UNIT_ASSERT_EQUAL(result, expected);
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestAsList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto pgmReturn = pb.AsList(pb.NewDataLiteral<ui32>(34));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestListIfTrue) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto pgmReturn = pb.ListIf(pb.NewDataLiteral(true),
+ pb.NewDataLiteral<ui32>(34));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 34);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestListIfFalse) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto pgmReturn = pb.ListIf(pb.NewDataLiteral(false),
+ pb.NewDataLiteral<ui32>(34));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ UNIT_ASSERT(!iterator.Skip());
+ UNIT_ASSERT(!iterator.Skip());
+ }
+
+ Y_UNIT_TEST_LLVM(TestNth) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto element0 = pb.NewDataLiteral<ui32>(34);
+ const auto element1 = pb.NewDataLiteral<ui32>(56);
+ const auto tuple = pb.NewTuple({element0, element1});
+ const auto pgmReturn = pb.Nth(tuple, 1);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue().template Get<ui32>();
+ UNIT_ASSERT_VALUES_EQUAL(res, 56);
+ }
+
+ Y_UNIT_TEST_LLVM(NonDeterministicEnv) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<float>(1);
+ const auto data2 = pb.NewDataLiteral<float>(2);
+ const auto data3 = pb.NewDataLiteral<float>(3);
+ const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<float>::Id), {data1, data2, data3});
+ const auto pgmReturn = pb.Sort(list,
+ pb.NewDataLiteral(false),
+ [](TRuntimeNode item) { return item; }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::abs(item.template Get<float>() - 3.0) < 0.001);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::abs(item.template Get<float>() - 2.0) < 0.001);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::abs(item.template Get<float>() - 1.0) < 0.001);
+ UNIT_ASSERT(false == iterator.Next(item));
+ UNIT_ASSERT(false == iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIndexDictContains) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto item1 = pb.NewDataLiteral<ui32>(7);
+ const auto item2 = pb.NewDataLiteral<ui32>(16);
+ const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {item1, item2});
+ const auto dict = pb.ToIndexDict(list);
+ const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto key0 = pb.NewDataLiteral<ui64>(0);
+ const auto key1 = pb.NewDataLiteral<ui64>(1);
+ const auto key2 = pb.NewDataLiteral<ui64>(2);
+ const auto keys = pb.NewList(type, {key0, key1, key2});
+ const auto pgmReturn = pb.Map(keys,
+ [&](TRuntimeNode key) { return pb.Contains(dict, key); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIndexDictLookup) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto item1 = pb.NewDataLiteral<i32>(7);
+ const auto item2 = pb.NewDataLiteral<i32>(16);
+ const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<i32>::Id), {item1, item2});
+ const auto dict = pb.ToIndexDict(list);
+ const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto key0 = pb.NewDataLiteral<ui64>(0);
+ const auto key1 = pb.NewDataLiteral<ui64>(1);
+ const auto key2 = pb.NewDataLiteral<ui64>(2);
+ const auto keys = pb.NewList(type, {key0, key1, key2});
+ const auto pgmReturn = pb.Map(keys,
+ [&](TRuntimeNode key) { return pb.Lookup(dict, key); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 16);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIndexDictContainsLazy) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto item1 = pb.NewDataLiteral<ui32>(7);
+ const auto item2 = pb.NewDataLiteral<ui32>(16);
+ const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {item1, item2});
+ const auto dict = pb.ToIndexDict(pb.LazyList(list));
+ const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto key0 = pb.NewDataLiteral<ui64>(0);
+ const auto key1 = pb.NewDataLiteral<ui64>(1);
+ const auto key2 = pb.NewDataLiteral<ui64>(2);
+ const auto keys = pb.NewList(type, {key0, key1, key2});
+ const auto pgmReturn = pb.Map(keys,
+ [&](TRuntimeNode key) { return pb.Contains(dict, key); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIndexDictLookupLazy) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto item1 = pb.NewDataLiteral<ui32>(7);
+ const auto item2 = pb.NewDataLiteral<ui32>(16);
+ const auto list = pb.NewList(pb.NewDataType(NUdf::TDataType<ui32>::Id), {item1, item2});
+ const auto dict = pb.ToIndexDict(pb.LazyList(list));
+ const auto type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto key0 = pb.NewDataLiteral<ui64>(0);
+ const auto key1 = pb.NewDataLiteral<ui64>(1);
+ const auto key2 = pb.NewDataLiteral<ui64>(2);
+ const auto keys = pb.NewList(type, {key0, key1, key2});
+ const auto pgmReturn = pb.Map(keys,
+ [&](TRuntimeNode key) { return pb.Lookup(dict, key); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 16);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestToBytes) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto data1 = pb.NewDataLiteral(0.f);
+ const auto data2 = pb.NewDataLiteral(-3.14f);
+ const auto data3 = pb.NewDataLiteral(-HUGE_VALF);
+ const auto data4 = pb.NewDataLiteral(HUGE_VALF);
+ const auto list = pb.NewList(type, {data1, data2, data3, data4});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToBytes(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x00\x00");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "\xc3\xf5\x48\xc0");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x80\xff");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x80\x7f");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestToBytesOpt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<float>::Id, true);
+ const auto data0 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<float>::Id);
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral(0.f));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral(-3.14f));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral(-HUGE_VALF));
+ const auto data4 = pb.NewOptional(pb.NewDataLiteral(HUGE_VALF));
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToBytes(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x00\x00");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "\xc3\xf5\x48\xc0");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x80\xff");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "\x00\x00\x80\x7f");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestToStringTemporarryUtf8) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("long prefix ");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("01234567890 long string");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("01234567890 very long string");
+ const auto list = pb.NewList(type, {data1, data2});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToString(pb.Concat(data0, item));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "long prefix 01234567890 long string");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "long prefix 01234567890 very long string");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFromString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("234");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto list = pb.NewList(type, {data1, data2});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.FromString(item, pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 234);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestStrictFromString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TVector<TRuntimeNode> tupleItems;
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("234");
+ tupleItems.push_back(pb.StrictFromString(data1, pb.NewDataType(NUdf::TDataType<ui32>::Id)));
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-1");
+ tupleItems.push_back(pb.StrictFromString(data2, pb.NewDataType(NUdf::TDataType<i32>::Id)));
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("3.1415926");
+ tupleItems.push_back(pb.StrictFromString(data3, pb.NewDataType(NUdf::TDataType<double>::Id)));
+ const auto pgmReturn = pb.NewTuple(tupleItems);
+
+ {
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 234);
+ UNIT_ASSERT_VALUES_EQUAL(res.GetElement(1).template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(res.GetElement(2).template Get<double>(), 3.1415926);
+ }
+
+ {
+ const auto wrongData = pb.NewDataLiteral<NUdf::EDataSlot::String>("vgfsbhj");
+ const auto fail = pb.StrictFromString(wrongData, pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ const auto failgraph = setup.BuildGraph(fail, {});
+ UNIT_ASSERT_EXCEPTION(failgraph->GetValue(), yexception);
+ }
+
+ {
+ const auto wrongData = pb.NewDataLiteral<NUdf::EDataSlot::String>("");
+ const auto fail = pb.StrictFromString(wrongData, pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto failgraph = setup.BuildGraph(fail, {});
+ UNIT_ASSERT_EXCEPTION(failgraph->GetValue(), yexception);
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestFromBytes) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TVector<TRuntimeNode> tupleItems;
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>(TString("\xEA\x00\x00\x00", 4));
+ tupleItems.push_back(pb.FromBytes(data1, NUdf::TDataType<ui32>::Id));
+ const auto data2 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<const char*>::Id);
+ tupleItems.push_back(pb.FromBytes(data2, NUdf::TDataType<ui32>::Id));
+ const auto pgmReturn = pb.NewTuple(tupleItems);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ UNIT_ASSERT(res.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 234);
+ UNIT_ASSERT(!res.GetElement(1));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMTRand) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto seed = pb.NewDataLiteral<ui64>(42);
+ auto pgmReturn = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui64>::Id));
+ auto rnd = pb.NewMTRand(seed);
+ const ui32 n = 5;
+ const ui64 expectedValues[n] = {
+ 13930160852258120406ull,
+ 11788048577503494824ull,
+ 13874630024467741450ull,
+ 2513787319205155662ull,
+ 16662371453428439381ull,
+ };
+ for (ui32 i = 0; i < n; ++i) {
+ const auto pair = pb.NextMTRand(rnd);
+ pgmReturn = pb.Append(pgmReturn, pb.Nth(pair, 0));
+ rnd = pb.Nth(pair, 1);
+ }
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ for (ui32 i = 0; i < n; ++i) {
+ UNIT_ASSERT(iterator.Next(item));
+ const ui64 value = item.template Get<ui64>();
+ //Cout << value << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value, expectedValues[i]);
+ }
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestRandom) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const double expectedValue1 = 0.13387664401253274;
+ const ui64 expectedValue2 = 2516265689700432462;
+
+ const auto rnd1 = pb.Random({});
+ const auto rnd2 = pb.RandomNumber({});
+ TVector<TRuntimeNode> args;
+ args.push_back(rnd1);
+ args.push_back(rnd2);
+ const auto pgmReturn = pb.NewTuple(args);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto tuple = graph->GetValue();
+ UNIT_ASSERT_DOUBLES_EQUAL(tuple.GetElement(0).template Get<double>(), expectedValue1, 1e-3);
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).template Get<ui64>(), expectedValue2);
+ }
+
+ Y_UNIT_TEST_LLVM(TestNow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const ui64 expectedValue = 10000000000000;
+
+ const auto ts = pb.Now({});
+ TVector<TRuntimeNode> args;
+ args.push_back(ts);
+ const auto pgmReturn = pb.NewTuple(args);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto tuple = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).template Get<ui64>(), expectedValue);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipAndTakeOverStream) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<ui8>::Id);
+ const auto data0 = pb.NewDataLiteral<ui8>(0);
+ const auto data1 = pb.NewDataLiteral<ui8>(1);
+ const auto data2 = pb.NewDataLiteral<ui8>(2);
+ const auto data3 = pb.NewDataLiteral<ui8>(3);
+ const auto data4 = pb.NewDataLiteral<ui8>(4);
+ const auto data5 = pb.NewDataLiteral<ui8>(5);
+ const auto data6 = pb.NewDataLiteral<ui8>(6);
+ const auto data7 = pb.NewDataLiteral<ui8>(7);
+ const auto data8 = pb.NewDataLiteral<ui8>(8);
+ const auto data9 = pb.NewDataLiteral<ui8>(9);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Take(pb.Skip(pb.Iterator(list, {}), pb.NewDataLiteral<ui64>(4ULL)), pb.NewDataLiteral<ui64>(3ULL));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 5);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 6);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipAndTakeOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<ui8>::Id);
+ const auto data0 = pb.NewDataLiteral<ui8>(0);
+ const auto data1 = pb.NewDataLiteral<ui8>(1);
+ const auto data2 = pb.NewDataLiteral<ui8>(2);
+ const auto data3 = pb.NewDataLiteral<ui8>(3);
+ const auto data4 = pb.NewDataLiteral<ui8>(4);
+ const auto data5 = pb.NewDataLiteral<ui8>(5);
+ const auto data6 = pb.NewDataLiteral<ui8>(6);
+ const auto data7 = pb.NewDataLiteral<ui8>(7);
+ const auto data8 = pb.NewDataLiteral<ui8>(8);
+ const auto data9 = pb.NewDataLiteral<ui8>(9);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.FromFlow(pb.Take(pb.Skip(pb.ToFlow(list), pb.NewDataLiteral<ui64>(4ULL)), pb.NewDataLiteral<ui64>(3ULL)));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 5);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui8>(), 6);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestLazyListFromArray) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto data0 = pb.NewDataLiteral<ui32>(1U);
+ const auto data1 = pb.NewDataLiteral<ui32>(2U);
+ const auto data2 = pb.NewDataLiteral<ui32>(3U);
+ const auto array = pb.NewList(type, {data0, data1, data2});
+
+ const auto pgmReturn = pb.LazyList(array);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto list = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 3U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElements(), nullptr);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0).template Get<ui32>(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1).template Get<ui32>(), 2U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(2).template Get<ui32>(), 3U);
+ }
+
+ Y_UNIT_TEST_LLVM(TestCollectLazyList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto data0 = pb.NewDataLiteral<ui32>(1U);
+ const auto data1 = pb.NewDataLiteral<ui32>(2U);
+ const auto data2 = pb.NewDataLiteral<ui32>(3U);
+ const auto array = pb.NewList(type, {data0, data1, data2});
+
+ const auto pgmReturn = pb.Collect(pb.LazyList(array));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto list = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 3U);
+ UNIT_ASSERT(list.GetElements());
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0).template Get<ui32>(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1).template Get<ui32>(), 2U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(2).template Get<ui32>(), 3U);
+ }
+
+ Y_UNIT_TEST_LLVM(TestAddAllTimezones) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto zones = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(1000U), pb.NewDataLiteral<ui16>(1U));
+ const auto pgmReturn = pb.Collect(pb.Map(zones, [&](TRuntimeNode id){ return pb.AddTimezone(pb.CurrentUtcDate({id}), id); }));
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto list = graph->GetValue();
+ UNIT_ASSERT(list.GetElement(0));
+ UNIT_ASSERT(list.GetElement(1));
+ UNIT_ASSERT(list.GetElement(2));
+ UNIT_ASSERT(list.GetElement(585));
+ UNIT_ASSERT(!list.GetElement(586));
+ UNIT_ASSERT(list.GetElement(587));
+ UNIT_ASSERT(list.GetElement(592));
+ UNIT_ASSERT(!list.GetElement(593));
+ UNIT_ASSERT(list.GetElement(594));
+ }
+
+ Y_UNIT_TEST_LLVM(TestRemoveTimezone) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("2019-10-24T13:01:37,Zulu");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("2019-10-24T13:01:37,Japan");
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("2019-10-24T13:01:37,Jamaica");
+
+ const auto datetimeType = pb.NewDataType(NUdf::EDataSlot::Datetime, true);
+ const auto datetimeTypeTz = pb.NewDataType(NUdf::EDataSlot::TzDatetime, true);
+
+ const auto list = pb.NewList(type, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode data) {
+ return pb.Convert(pb.FromString(data, datetimeTypeTz), datetimeType);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), 3ULL);
+
+ UNIT_ASSERT(!value.GetElement(0U).GetTimezoneId());
+ UNIT_ASSERT(!value.GetElement(1U).GetTimezoneId());
+ UNIT_ASSERT(!value.GetElement(2U).GetTimezoneId());
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0U).template Get<ui32>(), 1571922097U);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1U).template Get<ui32>(), 1571889697U);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2U).template Get<ui32>(), 1571940097U);
+ }
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 25u
+ Y_UNIT_TEST_LLVM(TestSqueezeToList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0);
+ const auto data1 = pb.NewDataLiteral(1.1);
+ const auto data2 = pb.NewDataLiteral(-3.14);
+ const auto data3 = pb.NewDataLiteral(121324.323);
+ const auto data4 = pb.NewDataLiteral(-7898.8);
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.FromFlow(pb.SqueezeToList(pb.ToFlow(list), pb.NewDataLiteral<ui64>(1000ULL)));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue full;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(full));
+ NUdf::TUnboxedValue stub;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(stub));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(stub));
+
+ UNIT_ASSERT_VALUES_EQUAL(full.GetListLength(), 5ULL);
+ UNIT_ASSERT_VALUES_EQUAL(full.GetElement(0U).template Get<double>(), 0.0);
+ UNIT_ASSERT_VALUES_EQUAL(full.GetElement(1U).template Get<double>(), 1.1);
+ UNIT_ASSERT_VALUES_EQUAL(full.GetElement(2U).template Get<double>(), -3.14);
+ UNIT_ASSERT_VALUES_EQUAL(full.GetElement(3U).template Get<double>(), 121324.323);
+ UNIT_ASSERT_VALUES_EQUAL(full.GetElement(4U).template Get<double>(), -7898.8);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSqueezeToListWithLimit) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral(0.0f);
+ const auto data1 = pb.NewDataLiteral(1.1f);
+ const auto data2 = pb.NewDataLiteral(-3.14f);
+ const auto data3 = pb.NewDataLiteral(12.323f);
+ const auto data4 = pb.NewDataLiteral(-7898.8f);
+ const auto type = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.FromFlow(pb.SqueezeToList(pb.ToFlow(list), pb.NewDataLiteral<ui64>(3ULL)));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue full;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(full));
+ NUdf::TUnboxedValue stub;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(stub));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(stub));
+
+ UNIT_ASSERT_VALUES_EQUAL(full.GetListLength(), 3ULL);
+ UNIT_ASSERT_VALUES_EQUAL(full.GetElement(0U).template Get<float>(), 0.0f);
+ UNIT_ASSERT_VALUES_EQUAL(full.GetElement(1U).template Get<float>(), 1.1f);
+ UNIT_ASSERT_VALUES_EQUAL(full.GetElement(2U).template Get<float>(), -3.14f);
+ }
+#endif
+ Y_UNIT_TEST_LLVM(TestPerfHolders) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto ui32Type = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto structType = pb.NewStructType({{"key", ui32Type}, {"value", strType}});
+ const auto mark = pb.NewDataLiteral<NUdf::EDataSlot::String>("!");
+
+ const auto listType = pb.NewListType(structType);
+ TCallableBuilder listRet(pb.GetTypeEnvironment(), "TestList", listType);
+ const auto listNode = listRet.Build();
+
+ const auto pgmReturn = pb.Map(pb.LazyList(TRuntimeNode(listNode, false)),
+ [&](TRuntimeNode item) {
+ return pb.NewStruct({
+ {"key", pb.Member(item, "key")},
+ {"value", pb.AggrConcat(mark, pb.Member(item, "value"))}
+ });
+ });
+
+ std::vector<ui32> src(10000U);
+ std::iota(src.begin(), src.end(), 0U);
+
+ const auto myStructFactory = [](const THolderFactory& factory, ui32 i) {
+ NUdf::TUnboxedValue* itemsPtr = nullptr;
+ const auto structObj = factory.CreateDirectArrayHolder(2U, itemsPtr);
+ itemsPtr[0] = NUdf::TUnboxedValuePod(ui32(i));
+ itemsPtr[1] = MakeString("ABCDEFGHIJKL");
+ return structObj;
+ };
+
+ const auto t1 = TInstant::Now();
+ {
+ const auto graph = setup.BuildGraph(pgmReturn, {listNode});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(src.size(), items));
+ std::transform(src.cbegin(), src.cend(), items, std::bind(myStructFactory, std::ref(graph->GetHolderFactory()), std::placeholders::_1));
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ ui32 i = 0U;
+ for (NUdf::TUnboxedValue current; iterator.Next(current); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(current.GetElement(0).template Get<ui32>(), i);
+ UNBOXED_VALUE_STR_EQUAL(current.GetElement(1), "!ABCDEFGHIJKL");
+ }
+ }
+
+ const auto t2 = TInstant::Now();
+ Cout << t2 - t1 << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestPerfGrep) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto ui32Type = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ auto structType = pb.NewEmptyStructType();
+ structType = pb.NewStructType(structType, "key", ui32Type);
+ structType = pb.NewStructType(structType, "value", strType);
+ const auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key");
+ const auto valueIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("value");
+
+ TRuntimeNode rowArg = pb.Arg(structType);
+
+ const auto pgmReturn = pb.Equals(pb.Member(rowArg, "value"),
+ pb.NewDataLiteral<NUdf::EDataSlot::String>("ABCDE"));
+
+ const auto t1 = TInstant::Now();
+ {
+ const auto graph = setup.BuildGraph(pgmReturn, {rowArg.GetNode()});
+ const auto row = graph->GetEntryPoint(0, true);
+
+ NUdf::TUnboxedValue* items = nullptr;
+ const auto structObj = graph->GetHolderFactory().CreateDirectArrayHolder(2, items);
+ graph->Prepare();
+ const ui32 n = 10000;
+ for (ui32 i = 0; i < n; ++i) {
+ items[keyIndex] = NUdf::TUnboxedValuePod(i);
+ items[valueIndex] = NUdf::TUnboxedValuePod::Embedded("ABCDF");
+ row->SetValue(graph->GetContext(), NUdf::TUnboxedValuePod(structObj));
+ const bool keep = graph->GetValue().template Get<bool>();
+ UNIT_ASSERT(!keep);
+ }
+ }
+
+ const auto t2 = TInstant::Now();
+ Cout << t2 - t1 << Endl;
+ }
+
+ Y_NO_INLINE NUdf::TUnboxedValuePod SpecialFunc(const NUdf::TUnboxedValuePod* args) {
+ const auto stringRef = args[1].AsStringRef();
+ return NUdf::TUnboxedValuePod(stringRef.Size() == 5 && std::memcmp(stringRef.Data(), "ABCDE", 5) == 0);
+ }
+
+ Y_UNIT_TEST_LLVM(TestPerfGrepSpecialFunc) {
+ const auto t1 = TInstant::Now();
+ {
+ NUdf::TUnboxedValuePod items[2];
+ const ui32 n = 10000;
+ for (ui32 i = 0; i < n; ++i) {
+ items[0] = NUdf::TUnboxedValuePod(i);
+ items[1] = NUdf::TUnboxedValuePod::Embedded("ABCDF");
+ bool keep = SpecialFunc(items).template Get<bool>();
+ UNIT_ASSERT(!keep);
+ }
+ }
+
+ const auto t2 = TInstant::Now();
+ Cout << t2 - t1 << Endl;
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp
new file mode 100644
index 00000000000..653d6c0cc08
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_condense_ut.cpp
@@ -0,0 +1,496 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLCondenseNodeTest) {
+ Y_UNIT_TEST_LLVM(TestSqueeze) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<double>(3.8));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<double>(-53.2));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<double>(233.8));
+ const auto data4 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<double>::Id);
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral<double>(HUGE_VAL));
+ const auto list = pb.NewList(dataType, {data4, data3, data2, data1});
+
+ const auto pgmReturn = pb.Squeeze(pb.Iterator(list, {}), data0,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrMin(item, state);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -53.2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSqueezeOnEmpty) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral<double>(HUGE_VAL));
+ const auto list = pb.NewEmptyList(dataType);
+
+ const auto pgmReturn = pb.Squeeze(pb.Iterator(list, {}), data0,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrMin(item, state);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), HUGE_VAL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSqueeze1OverEmpty) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewEmptyList(dataType);
+ const auto pgmReturn = pb.Squeeze1(pb.Iterator(list, {}),
+ [&](TRuntimeNode item) {
+ return pb.Minus(item);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Mul(item, state);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSqueeze1OverSingle) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto data1 = pb.NewDataLiteral<i32>(1);
+ const auto list = pb.NewList(dataType, {data1});
+ const auto pgmReturn = pb.Squeeze1(pb.Iterator(list, {}),
+ [&](TRuntimeNode item) {
+ return pb.Minus(item);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Mul(item, state);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1);
+
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSqueeze1OverMany) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto data1 = pb.NewDataLiteral<i32>(1);
+ const auto data2 = pb.NewDataLiteral<i32>(2);
+ const auto data3 = pb.NewDataLiteral<i32>(7);
+ const auto list = pb.NewList(dataType, {data1, data2, data3});
+ const auto pgmReturn = pb.Squeeze1(pb.Iterator(list, {}),
+ [&](TRuntimeNode item) {
+ return pb.Minus(item);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Mul(item, state);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -14);
+
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCondense) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral<double>(0.0));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<double>(3.8));
+ const auto data2 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<double>::Id);
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<double>(-53.2));
+ const auto data4 = pb.NewOptional(pb.NewDataLiteral<double>(233.8));
+ const auto data5 = pb.NewOptional(pb.NewDataLiteral<double>(3.14));
+ const auto data6 = pb.NewOptional(pb.NewDataLiteral<double>(-73.12));
+ const auto data7 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<double>::Id);
+ const auto data8 = pb.NewOptional(pb.NewDataLiteral<double>(233.8));
+ const auto data9 = pb.NewOptional(pb.NewDataLiteral<double>(1221.8));
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Condense(pb.Iterator(list, {}), data0,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Or({pb.Not(pb.Exists(item)), pb.Less(state, data0)});
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrAdd(item, state);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 3.8);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -53.2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 163.82);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 1455.6);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCondense1) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto data0 = pb.NewDataLiteral<i32>(-1);
+ const auto data1 = pb.NewDataLiteral<i32>(2);
+ const auto data2 = pb.NewDataLiteral<i32>(0);
+ const auto data3 = pb.NewDataLiteral<i32>(7);
+ const auto data4 = pb.NewDataLiteral<i32>(5);
+ const auto data5 = pb.NewDataLiteral<i32>(-7);
+ const auto data6 = pb.NewDataLiteral<i32>(-6);
+ const auto data7 = pb.NewDataLiteral<i32>(4);
+ const auto data8 = pb.NewDataLiteral<i32>(8);
+ const auto data9 = pb.NewDataLiteral<i32>(9);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+ const auto pgmReturn = pb.Condense1(pb.Iterator(list, {}),
+ [&](TRuntimeNode item) {
+ return pb.Minus(item);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.LessOrEqual(item, state);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Mul(item, state);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 6);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -288);
+
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCondenseOverList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral<double>(0.0));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<double>(3.8));
+ const auto data2 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<double>::Id);
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<double>(-53.2));
+ const auto data4 = pb.NewOptional(pb.NewDataLiteral<double>(233.8));
+ const auto data5 = pb.NewOptional(pb.NewDataLiteral<double>(3.14));
+ const auto data6 = pb.NewOptional(pb.NewDataLiteral<double>(-73.12));
+ const auto data7 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<double>::Id);
+ const auto data8 = pb.NewOptional(pb.NewDataLiteral<double>(233.8));
+ const auto data9 = pb.NewOptional(pb.NewDataLiteral<double>(1221.8));
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Condense(list, data0,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Or({pb.Not(pb.Exists(item)), pb.Less(state, data0)});
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrAdd(item, state);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 3.8);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -53.2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 163.82);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 1455.6);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCondense1OverList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto data0 = pb.NewDataLiteral<i32>(-1);
+ const auto data1 = pb.NewDataLiteral<i32>(2);
+ const auto data2 = pb.NewDataLiteral<i32>(0);
+ const auto data3 = pb.NewDataLiteral<i32>(7);
+ const auto data4 = pb.NewDataLiteral<i32>(5);
+ const auto data5 = pb.NewDataLiteral<i32>(-7);
+ const auto data6 = pb.NewDataLiteral<i32>(-6);
+ const auto data7 = pb.NewDataLiteral<i32>(4);
+ const auto data8 = pb.NewDataLiteral<i32>(8);
+ const auto data9 = pb.NewDataLiteral<i32>(9);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+ const auto pgmReturn = pb.Condense1(list,
+ [&](TRuntimeNode item) {
+ return pb.Minus(item);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.LessOrEqual(item, state);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Mul(item, state);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 6);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -288);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCondenseInterrupt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id));
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral<bool>(false));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<bool>(false));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<bool>(false));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<bool>(false));
+ const auto data4 = pb.NewOptional(pb.NewDataLiteral<bool>(true));
+ const auto data5 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id);
+ const auto data6 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6});
+
+ const auto pgmReturn = pb.FromFlow(pb.Condense(pb.ToFlow(list), pb.NewDataLiteral<bool>(false),
+ [&](TRuntimeNode, TRuntimeNode state) {
+ return pb.If(state, pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id), pb.NewDataLiteral<bool>(false));
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Or({pb.Unwrap(item, pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), state});
+ }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(item.template Get<bool>());
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCondense1Interrupt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id));
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral<bool>(true));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<bool>(true));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<bool>(true));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<bool>(true));
+ const auto data4 = pb.NewOptional(pb.NewDataLiteral<bool>(false));
+ const auto data5 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id);
+ const auto data6 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6});
+
+ const auto pgmReturn = pb.FromFlow(pb.Condense1(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return pb.Unwrap(item, pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ },
+ [&](TRuntimeNode, TRuntimeNode state) {
+ return pb.If(state, pb.NewDataLiteral<bool>(false), pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id));
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.And({pb.Unwrap(item, pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), state});
+ }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.template Get<bool>());
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCondenseInterruptEndlessStream) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto pgmReturn = pb.Condense(pb.SourceOf(pb.NewStreamType(pb.NewNull().GetStaticType())), pb.NewDataLiteral<ui32>(0U),
+ [&](TRuntimeNode, TRuntimeNode state) {
+ return pb.If(pb.AggrLess(state, pb.NewDataLiteral<ui32>(123456U)), pb.NewDataLiteral<bool>(false), pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id));
+ },
+ [&](TRuntimeNode, TRuntimeNode state) {
+ return pb.AggrAdd(pb.NewDataLiteral<ui32>(1U), state);
+ }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 123456U);
+
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCondense1InterruptEndlessFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto pgmReturn = pb.FromFlow(pb.Condense1(pb.SourceOf(pb.NewFlowType(pb.NewNull().GetStaticType())),
+ [&](TRuntimeNode) {
+ return pb.NewDataLiteral<ui32>(0U);
+ },
+ [&](TRuntimeNode, TRuntimeNode state) {
+ return pb.If(pb.AggrLess(state, pb.NewDataLiteral<ui32>(123456U)), pb.NewDataLiteral<bool>(false), pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id));
+ },
+ [&](TRuntimeNode, TRuntimeNode state) {
+ return pb.AggrAdd(pb.NewDataLiteral<ui32>(1U), state);
+ }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 123456U);
+
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+
+ Y_UNIT_TEST_LLVM(TestCondenseListeralListInMap) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("other");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("foo");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("bar");
+ const auto type = pb.NewDataType(NUdf::EDataSlot::String);
+ const auto list2 = pb.NewList(type, {data1, data2});
+ const auto list0 = pb.NewList(type, {data1, data2, data0});
+
+ const auto pgmReturn = pb.Map(list0,
+ [&](TRuntimeNode item) {
+ return pb.Head(pb.Condense(list2,
+ pb.NewDataLiteral(false),
+ [&](TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral(false); },
+ [&](TRuntimeNode it, TRuntimeNode state) { return pb.Or({state, pb.AggrEquals(item, it)}); }
+ ));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.Get<bool>());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.Get<bool>());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.Get<bool>());
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCondense1ListeralListInMap) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("other");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("foo");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("bar");
+ const auto type = pb.NewDataType(NUdf::EDataSlot::String);
+ const auto list2 = pb.NewList(type, {data1, data2});
+ const auto list0 = pb.NewList(type, {data1, data2, data0});
+
+ const auto pgmReturn = pb.Map(list0,
+ [&](TRuntimeNode item) {
+ return pb.Head(pb.Condense1(list2,
+ [&](TRuntimeNode it) { return pb.AggrEquals(item, it); },
+ [&](TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral(false); },
+ [&](TRuntimeNode it, TRuntimeNode state) { return pb.Or({state, pb.AggrEquals(item, it)}); }
+ ));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.Get<bool>());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.Get<bool>());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.Get<bool>());
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp
new file mode 100644
index 00000000000..00fa7eca1b6
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_decimal_ut.cpp
@@ -0,0 +1,2329 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLDecimalTest) {
+ Y_UNIT_TEST_LLVM(TestNanvl) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewOptionalType(pb.NewDecimalType(13, 5));
+ const auto data0 = pb.NewOptional(pb.NewDecimalLiteral(0, 13, 5));
+ const auto data1 = pb.NewOptional(pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 13, 5));
+ const auto data2 = pb.NewOptional(pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 13, 5));
+ const auto data3 = pb.NewOptional(pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 13, 5));
+ const auto data4 = pb.NewEmptyOptional(type);
+ const auto data5 = pb.NewOptional(pb.NewDecimalLiteral(-NYql::NDecimal::Nan(), 13, 5));
+ const auto data = pb.NewDecimalLiteral(314159, 13, 5);
+
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Nanvl(item, data);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 314159);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == +NYql::NDecimal::Inf());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 314159);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestToIntegral) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDecimalType(13, 1);
+ const auto data0 = pb.NewDecimalLiteral(0, 13, 1);
+ const auto data1 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 13, 1);
+ const auto data2 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 13, 1);
+ const auto data3 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 13, 1);
+ const auto data4 = pb.NewDecimalLiteral(1270, 13, 1);
+ const auto data5 = pb.NewDecimalLiteral(-1280, 13, 1);
+ const auto data6 = pb.NewDecimalLiteral(2550, 13, 1);
+ const auto data7 = pb.NewDecimalLiteral(-2560, 13, 1);
+ const auto data8 = pb.NewDecimalLiteral(2560, 13, 1);
+ const auto data9 = pb.NewDecimalLiteral(-2570, 13, 1);
+ const auto dataA = pb.NewDecimalLiteral(327670, 13, 1);
+ const auto dataB = pb.NewDecimalLiteral(-327680, 13, 1);
+ const auto dataC = pb.NewDecimalLiteral(655350, 13, 1);
+ const auto dataD = pb.NewDecimalLiteral(-655360, 13, 1);
+ const auto dataE = pb.NewDecimalLiteral(21474836470, 13, 1);
+ const auto dataF = pb.NewDecimalLiteral(-21474836480, 13, 1);
+ const auto dataG = pb.NewDecimalLiteral(21474836480, 13, 1);
+ const auto dataH = pb.NewDecimalLiteral(-21474836490, 13, 1);
+
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9, dataA, dataB, dataC, dataD, dataE, dataF, dataG, dataH});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i8>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui8>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i16>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui16>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i32>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui32>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<i64>::Id, true)),
+ pb.ToIntegral(item, pb.NewDataType(NUdf::TDataType<ui64>::Id, true))
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 0);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT(!item.GetElements()[4]);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT(!item.GetElements()[6]);
+ UNIT_ASSERT(!item.GetElements()[7]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT(!item.GetElements()[4]);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT(!item.GetElements()[6]);
+ UNIT_ASSERT(!item.GetElements()[7]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT(!item.GetElements()[4]);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT(!item.GetElements()[6]);
+ UNIT_ASSERT(!item.GetElements()[7]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), 127);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 127);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 127);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 127);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 127);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 127);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 127);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 127);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[0].Get<i8>(), -128);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), -128);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -128);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -128);
+ UNIT_ASSERT(!item.GetElements()[7]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[1].Get<ui8>(), 255);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 255);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 255);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 255);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 255);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 255);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 255);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), -256);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -256);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -256);
+ UNIT_ASSERT(!item.GetElements()[7]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 256);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 256);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 256);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 256);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 256);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 256);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), -257);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -257);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -257);
+ UNIT_ASSERT(!item.GetElements()[7]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), 32767);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 32767);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 32767);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 32767);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 32767);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 32767);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[2].Get<i16>(), -32768);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -32768);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -32768);
+ UNIT_ASSERT(!item.GetElements()[7]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[3].Get<ui16>(), 65535);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 65535);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 65535);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 65535);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 65535);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -65536);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -65536);
+ UNIT_ASSERT(!item.GetElements()[7]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), 2147483647LL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 2147483647LL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 2147483647LL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 2147483647LL);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[4].Get<i32>(), -2147483648LL);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -2147483648LL);
+ UNIT_ASSERT(!item.GetElements()[7]);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT(!item.GetElements()[4]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[5].Get<ui32>(), 2147483648LL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), 2147483648LL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[7].Get<ui64>(), 2147483648LL);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElements()[0]);
+ UNIT_ASSERT(!item.GetElements()[1]);
+ UNIT_ASSERT(!item.GetElements()[2]);
+ UNIT_ASSERT(!item.GetElements()[3]);
+ UNIT_ASSERT(!item.GetElements()[4]);
+ UNIT_ASSERT(!item.GetElements()[5]);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElements()[6].Get<i64>(), -2147483649LL);
+ UNIT_ASSERT(!item.GetElements()[7]);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestToFloat) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDecimalType(10, 3);
+ const auto data0 = pb.NewDecimalLiteral(2123, 10, 3);
+ const auto data1 = pb.NewDecimalLiteral(233, 10, 3);
+ const auto data2 = pb.NewDecimalLiteral(0, 10, 3);
+ const auto data3 = pb.NewDecimalLiteral(-3277823, 10, 3);
+ const auto data4 = pb.NewDecimalLiteral(-1, 10, 3);
+ const auto data5 = pb.NewDecimalLiteral(7128, 10, 3);
+ const auto data6 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 3);
+ const auto data7 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 3);
+ const auto data8 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 3);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Convert(item, pb.NewDataType(NUdf::TDataType<float>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 2.123f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.233f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 0.0f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -3277.823f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -0.001f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), 7.128f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::isnan(item.template Get<float>()));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::isinf(item.template Get<float>()) && item.template Get<float>() > 0.0f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::isinf(item.template Get<float>()) && item.template Get<float>() < 0.0f);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestToDouble) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDecimalType(10, 5);
+ const auto data0 = pb.NewDecimalLiteral(2123, 10, 5);
+ const auto data1 = pb.NewDecimalLiteral(233, 10, 5);
+ const auto data2 = pb.NewDecimalLiteral(0, 10, 5);
+ const auto data3 = pb.NewDecimalLiteral(-3277823, 10, 5);
+ const auto data4 = pb.NewDecimalLiteral(-1, 10, 5);
+ const auto data5 = pb.NewDecimalLiteral(7128, 10, 5);
+ const auto data6 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 5);
+ const auto data7 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 5);
+ const auto data8 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 5);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Convert(item, pb.NewDataType(NUdf::TDataType<double>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.02123);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.00233);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -32.77823);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), -0.00001);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<double>(), 0.07128);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::isnan(item.template Get<double>()));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::isinf(item.template Get<double>()) && item.template Get<double>() > 0.0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(std::isinf(item.template Get<double>()) && item.template Get<double>() < 0.0);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDiv) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDecimalType(10, 0);
+ const auto data0 = pb.NewDecimalLiteral(2, 10, 0);
+ const auto data1 = pb.NewDecimalLiteral(23, 10, 0);
+ const auto data2 = pb.NewDecimalLiteral(-23, 10, 0);
+ const auto data3 = pb.NewDecimalLiteral(25, 10, 0);
+ const auto data4 = pb.NewDecimalLiteral(-25, 10, 0);
+ const auto data5 = pb.NewDecimalLiteral(1, 10, 0);
+ const auto data6 = pb.NewDecimalLiteral(-1, 10, 0);
+ const auto data7 = pb.NewDecimalLiteral(3, 10, 0);
+ const auto data8 = pb.NewDecimalLiteral(-3, 10, 0);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.DecimalDiv(item, data0);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 12);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -12);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 12);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -12);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDivInt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i8>::Id);
+ const auto data0 = pb.NewDecimalLiteral(-238973, 9, 3);
+ const auto data1 = pb.NewDataLiteral<i8>(0);
+ const auto data2 = pb.NewDataLiteral<i8>(-1);
+ const auto data3 = pb.NewDataLiteral<i8>(-128);
+ const auto data4 = pb.NewDataLiteral<i8>(3);
+ const auto data5 = pb.NewDataLiteral<i8>(5);
+ const auto data6 = pb.NewDataLiteral<i8>(-7);
+ const auto data7 = pb.NewDataLiteral<i8>(13);
+ const auto data8 = pb.NewDataLiteral<i8>(-19);
+ const auto data9 = pb.NewDataLiteral<i8>(42);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.DecimalDiv(data0, item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 238973);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 1866);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -79658);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -47795);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 34139);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -18383);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 12577);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -5690);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMod) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDecimalType(5, 2);
+ const auto data0 = pb.NewDecimalLiteral(-12323, 5, 2);
+ const auto data1 = pb.NewDecimalLiteral(0, 5, 2);
+ const auto data2 = pb.NewDecimalLiteral(NYql::NDecimal::Inf(), 5, 2);
+ const auto data3 = pb.NewDecimalLiteral(-1, 5, 2);
+ const auto data4 = pb.NewDecimalLiteral(2, 5, 2);
+ const auto data5 = pb.NewDecimalLiteral(-3, 5, 2);
+ const auto data6 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 5, 2);
+ const auto data7 = pb.NewDecimalLiteral(7, 5, 2);
+ const auto data8 = pb.NewDecimalLiteral(-10000, 5, 2);
+ const auto data9 = pb.NewDecimalLiteral(12329, 5, 2);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.DecimalMod(data0, item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -2323);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -12323);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestModInt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i16>::Id);
+ const auto data0 = pb.NewDecimalLiteral(-743, 3, 2);
+ const auto data1 = pb.NewDataLiteral<i16>(0);
+ const auto data2 = pb.NewDataLiteral<i16>(1);
+ const auto data3 = pb.NewDataLiteral<i16>(-2);
+ const auto data4 = pb.NewDataLiteral<i16>(3);
+ const auto data5 = pb.NewDataLiteral<i16>(4);
+ const auto data6 = pb.NewDataLiteral<i16>(-5);
+ const auto data7 = pb.NewDataLiteral<i16>(8);
+ const auto data8 = pb.NewDataLiteral<i16>(10);
+ const auto data9 = pb.NewDataLiteral<i16>(-10);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.DecimalMod(data0, item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -43);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -143);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -143);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -343);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -243);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -743);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -743);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -743);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMul) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDecimalType(10, 2);
+ const auto data0 = pb.NewDecimalLiteral(333, 10, 2);
+ const auto data1 = pb.NewDecimalLiteral(-100, 10, 2);
+ const auto data2 = pb.NewDecimalLiteral(-120, 10, 2);
+ const auto data3 = pb.NewDecimalLiteral(3, 10, 2);
+ const auto data4 = pb.NewDecimalLiteral(77, 10, 2);
+ const auto data5 = pb.NewDecimalLiteral(122, 10, 2);
+ const auto data6 = pb.NewDecimalLiteral(1223, 10, 2);
+ const auto data7 = pb.NewDecimalLiteral(-999, 10, 2);
+ const auto data8 = pb.NewDecimalLiteral(0, 10, 2);
+ const auto data9 = pb.NewDecimalLiteral(-3003003003LL, 10, 2);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.DecimalMul(item, data0);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 1109);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -333);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -400);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 10);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 256);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 406);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 4073);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -3327);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMulUInt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id);
+ const auto data0 = pb.NewDecimalLiteral(-333, 7, 2);
+ const auto data1 = pb.NewDataLiteral<ui16>(0);
+ const auto data2 = pb.NewDataLiteral<ui16>(1);
+ const auto data3 = pb.NewDataLiteral<ui16>(2);
+ const auto data4 = pb.NewDataLiteral<ui16>(3);
+ const auto data5 = pb.NewDataLiteral<ui16>(10);
+ const auto data6 = pb.NewDataLiteral<ui16>(100);
+ const auto data7 = pb.NewDataLiteral<ui16>(1000);
+ const auto data8 = pb.NewDataLiteral<ui16>(10000);
+ const auto data9 = pb.NewDataLiteral<ui16>(65535);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.DecimalMul(data0, item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -333);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -666);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -999);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -3330);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -33300);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -333000);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -3330000);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMulTinyInt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i8>::Id);
+ const auto data0 = pb.NewDecimalLiteral(3631400, 32, 4);
+ const auto data1 = pb.NewDataLiteral<i8>(0);
+ const auto data2 = pb.NewDataLiteral<i8>(1);
+ const auto data3 = pb.NewDataLiteral<i8>(-1);
+ const auto data4 = pb.NewDataLiteral<i8>(3);
+ const auto data5 = pb.NewDataLiteral<i8>(-3);
+ const auto data6 = pb.NewDataLiteral<i8>(100);
+ const auto data7 = pb.NewDataLiteral<i8>(-100);
+ const auto data8 = pb.NewDataLiteral<i8>(127);
+ const auto data9 = pb.NewDataLiteral<i8>(-128);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.DecimalMul(data0, item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 3631400);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -3631400);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 10894200);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -10894200);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 363140000);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -363140000);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 461187800);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -464819200);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCastAndMulTinyInt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDecimalType(32, 4);
+ const auto data0 = pb.NewDataLiteral<i8>(1);
+ const auto data1 = pb.NewDecimalLiteral(3145926, 32, 4);
+ const auto data2 = pb.NewDecimalLiteral(-3145926, 32, 4);
+ const auto list = pb.NewList(dataType, {data1, data2});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.DecimalMul(item, data0), pb.DecimalMul(item, pb.ToDecimal(data0, 32, 4))});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 3145926);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 3145926);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -3145926);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -3145926);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestLongintMul) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDecimalType(10, 0);
+ const auto data0 = pb.NewDecimalLiteral(333, 10, 0);
+ const auto data1 = pb.NewDecimalLiteral(-100, 10, 0);
+ const auto data2 = pb.NewDecimalLiteral(-120, 10, 0);
+ const auto data3 = pb.NewDecimalLiteral(3, 10, 0);
+ const auto data4 = pb.NewDecimalLiteral(77, 10, 0);
+ const auto data5 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 0);
+ const auto data6 = pb.NewDecimalLiteral(30030031, 10, 0);
+ const auto data7 = pb.NewDecimalLiteral(-30030031, 10, 0);
+ const auto data8 = pb.NewDecimalLiteral(0, 10, 0);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.DecimalMul(item, data0);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 110889);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -33300);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -39960);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 999);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 25641);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Inf());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestScaleUp) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDecimalType(10, 2);
+ const auto data0 = pb.NewDecimalLiteral(333, 10, 2);
+ const auto data1 = pb.NewDecimalLiteral(-100, 10, 2);
+ const auto data2 = pb.NewDecimalLiteral(-120, 10, 2);
+ const auto data3 = pb.NewDecimalLiteral(3, 10, 2);
+ const auto data4 = pb.NewDecimalLiteral(77, 10, 2);
+ const auto data5 = pb.NewDecimalLiteral(122, 10, 2);
+ const auto data6 = pb.NewDecimalLiteral(1223, 10, 2);
+ const auto data7 = pb.NewDecimalLiteral(-999, 10, 2);
+ const auto data8 = pb.NewDecimalLiteral(0, 10, 2);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToDecimal(item, 12, 4);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 33300);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -10000);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -12000);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 300);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 7700);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 12200);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 122300);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -99900);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestScaleDown) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDecimalType(10, 2);
+ const auto data0 = pb.NewDecimalLiteral(-251, 10, 2);
+ const auto data1 = pb.NewDecimalLiteral(-250, 10, 2);
+ const auto data2 = pb.NewDecimalLiteral(-150, 10, 2);
+ const auto data3 = pb.NewDecimalLiteral(-51, 10, 2);
+ const auto data4 = pb.NewDecimalLiteral(50, 10, 2);
+ const auto data5 = pb.NewDecimalLiteral(50, 10, 2);
+ const auto data6 = pb.NewDecimalLiteral(51, 10, 2);
+ const auto data7 = pb.NewDecimalLiteral(150, 10, 2);
+ const auto data8 = pb.NewDecimalLiteral(250, 10, 2);
+ const auto data9 = pb.NewDecimalLiteral(251, 10, 2);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToDecimal(item, 8, 0);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 3);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMax) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDecimalLiteral(-NYql::NDecimal::Nan(), 13, 2);
+ const auto data2 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 13, 2);
+ const auto data3 = pb.NewDecimalLiteral(314, 13, 2);
+ const auto data4 = pb.NewDecimalLiteral(-213, 13, 2);
+ const auto data5 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 13, 2);
+ const auto dataType = pb.NewDecimalType(13, 2);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5});
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode left) {
+ return pb.Map(list,
+ [&](TRuntimeNode right) {
+ return pb.NewTuple({pb.Min(left, right), pb.Max(left, right)});
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 314);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -213);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 314);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 314);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 314);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 314);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -213);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -213);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -213);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -213);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestAggrMinMax) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 13, 2);
+ const auto data2 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 13, 2);
+ const auto data3 = pb.NewDecimalLiteral(314, 13, 2);
+ const auto data4 = pb.NewDecimalLiteral(-213, 13, 2);
+ const auto data5 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 13, 2);
+ const auto dataType = pb.NewDecimalType(13, 2);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5});
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode left) {
+ return pb.Map(list,
+ [&](TRuntimeNode right) {
+ return pb.NewTuple({pb.AggrMin(left, right), pb.AggrMax(left, right)});
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 314);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 314);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 314);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 314);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 314);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -213);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -213);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -213);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 314);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -213);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestAddSub) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDecimalLiteral(-NYql::NDecimal::Nan(), 13, 2);
+ const auto data2 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 13, 2);
+ const auto data3 = pb.NewDecimalLiteral(314, 13, 2);
+ const auto data4 = pb.NewDecimalLiteral(-213, 13, 2);
+ const auto data5 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 13, 2);
+
+ const auto dataType = pb.NewDecimalType(13, 2);
+ const auto list = pb.NewList(dataType, {data1, data2, data3, data4, data5});
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode left) {
+ return pb.Map(list,
+ [&](TRuntimeNode right) {
+ return pb.NewTuple({pb.Add(left, right), pb.Sub(left, right)});
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 628);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 0);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 101);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 527);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 101);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -527);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -426);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 0);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCompares) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1l = pb.NewDecimalLiteral(-7, 10, 0);
+ const auto data2l = pb.NewDecimalLiteral(3, 10, 0);
+ const auto data3l = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 0);
+ const auto data4l = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 0);
+ const auto data5l = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 0);
+
+ const auto data1r = pb.NewDecimalLiteral(-700, 7, 2);
+ const auto data2r = pb.NewDecimalLiteral(300, 7, 2);
+ const auto data3r = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 7, 2);
+ const auto data4r = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 7, 2);
+ const auto data5r = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 7, 2);
+
+ auto pairType = pb.NewTupleType({pb.NewDecimalType(10, 0), pb.NewDecimalType(7, 2)});
+ const auto list = pb.NewList(pairType, {
+ pb.NewTuple({data1l, data1r}),
+ pb.NewTuple({data1l, data2r}),
+ pb.NewTuple({data1l, data3r}),
+ pb.NewTuple({data1l, data4r}),
+ pb.NewTuple({data1l, data5r}),
+
+ pb.NewTuple({data2l, data1r}),
+ pb.NewTuple({data2l, data2r}),
+ pb.NewTuple({data2l, data3r}),
+ pb.NewTuple({data2l, data4r}),
+ pb.NewTuple({data2l, data5r}),
+
+ pb.NewTuple({data3l, data1r}),
+ pb.NewTuple({data3l, data2r}),
+ pb.NewTuple({data3l, data3r}),
+ pb.NewTuple({data3l, data4r}),
+ pb.NewTuple({data3l, data5r}),
+
+ pb.NewTuple({data4l, data1r}),
+ pb.NewTuple({data4l, data2r}),
+ pb.NewTuple({data4l, data3r}),
+ pb.NewTuple({data4l, data4r}),
+ pb.NewTuple({data4l, data5r}),
+
+ pb.NewTuple({data5l, data1r}),
+ pb.NewTuple({data5l, data2r}),
+ pb.NewTuple({data5l, data3r}),
+ pb.NewTuple({data5l, data4r}),
+ pb.NewTuple({data5l, data5r}),
+ });
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1))
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestComparesWithIntegral) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto optType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i64>::Id));
+
+ const auto data1l = pb.NewOptional(pb.NewDataLiteral<i64>(-7LL));
+ const auto data2l = pb.NewOptional(pb.NewDataLiteral<i64>(3LL));
+ const auto data3l = pb.NewEmptyOptional(optType);
+ const auto data4l = pb.NewOptional(pb.NewDataLiteral<i64>(std::numeric_limits<i64>::min()));
+ const auto data5l = pb.NewOptional(pb.NewDataLiteral<i64>(std::numeric_limits<i64>::max()));
+
+ const auto data1r = pb.NewDecimalLiteral(-7000000000000000000LL, 20, 18);
+ const auto data2r = pb.NewDecimalLiteral(3000000000000000000LL, 20, 18);
+ const auto data3r = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 20, 18);
+ const auto data4r = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 20, 18);
+ const auto data5r = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 20, 18);
+
+ auto pairType = pb.NewTupleType({optType, pb.NewDecimalType(20, 18)});
+ const auto list = pb.NewList(pairType, {
+ pb.NewTuple({data1l, data1r}),
+ pb.NewTuple({data1l, data2r}),
+ pb.NewTuple({data1l, data3r}),
+ pb.NewTuple({data1l, data4r}),
+ pb.NewTuple({data1l, data5r}),
+
+ pb.NewTuple({data2l, data1r}),
+ pb.NewTuple({data2l, data2r}),
+ pb.NewTuple({data2l, data3r}),
+ pb.NewTuple({data2l, data4r}),
+ pb.NewTuple({data2l, data5r}),
+
+ pb.NewTuple({data3l, data1r}),
+ pb.NewTuple({data3l, data2r}),
+ pb.NewTuple({data3l, data3r}),
+ pb.NewTuple({data3l, data4r}),
+ pb.NewTuple({data3l, data5r}),
+
+ pb.NewTuple({data4l, data1r}),
+ pb.NewTuple({data4l, data2r}),
+ pb.NewTuple({data4l, data3r}),
+ pb.NewTuple({data4l, data4r}),
+ pb.NewTuple({data4l, data5r}),
+
+ pb.NewTuple({data5l, data1r}),
+ pb.NewTuple({data5l, data2r}),
+ pb.NewTuple({data5l, data3r}),
+ pb.NewTuple({data5l, data4r}),
+ pb.NewTuple({data5l, data5r}),
+ });
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.Equals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.NotEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Less(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.LessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.Greater(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.GreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1))
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0)); // ==
+ UNIT_ASSERT(!item.GetElement(1)); // !=
+ UNIT_ASSERT(!item.GetElement(2)); // <
+ UNIT_ASSERT(!item.GetElement(3)); // <=
+ UNIT_ASSERT(!item.GetElement(4)); // >
+ UNIT_ASSERT(!item.GetElement(5)); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0)); // ==
+ UNIT_ASSERT(!item.GetElement(1)); // !=
+ UNIT_ASSERT(!item.GetElement(2)); // <
+ UNIT_ASSERT(!item.GetElement(3)); // <=
+ UNIT_ASSERT(!item.GetElement(4)); // >
+ UNIT_ASSERT(!item.GetElement(5)); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0)); // ==
+ UNIT_ASSERT(!item.GetElement(1)); // !=
+ UNIT_ASSERT(!item.GetElement(2)); // <
+ UNIT_ASSERT(!item.GetElement(3)); // <=
+ UNIT_ASSERT(!item.GetElement(4)); // >
+ UNIT_ASSERT(!item.GetElement(5)); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0)); // ==
+ UNIT_ASSERT(!item.GetElement(1)); // !=
+ UNIT_ASSERT(!item.GetElement(2)); // <
+ UNIT_ASSERT(!item.GetElement(3)); // <=
+ UNIT_ASSERT(!item.GetElement(4)); // >
+ UNIT_ASSERT(!item.GetElement(5)); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0)); // ==
+ UNIT_ASSERT(!item.GetElement(1)); // !=
+ UNIT_ASSERT(!item.GetElement(2)); // <
+ UNIT_ASSERT(!item.GetElement(3)); // <=
+ UNIT_ASSERT(!item.GetElement(4)); // >
+ UNIT_ASSERT(!item.GetElement(5)); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestAggrCompares) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDecimalLiteral(-7, 10, 0);
+ const auto data2 = pb.NewDecimalLiteral(3, 10, 0);
+ const auto data3 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 0);
+ const auto data4 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 0);
+ const auto data5 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 0);
+
+ auto pairType = pb.NewTupleType({pb.NewDecimalType(10, 0), pb.NewDecimalType(10, 0)});
+ const auto list = pb.NewList(pairType, {
+ pb.NewTuple({data1, data1}),
+ pb.NewTuple({data1, data2}),
+ pb.NewTuple({data1, data3}),
+ pb.NewTuple({data1, data4}),
+ pb.NewTuple({data1, data5}),
+
+ pb.NewTuple({data2, data1}),
+ pb.NewTuple({data2, data2}),
+ pb.NewTuple({data2, data3}),
+ pb.NewTuple({data2, data4}),
+ pb.NewTuple({data2, data5}),
+
+ pb.NewTuple({data3, data1}),
+ pb.NewTuple({data3, data2}),
+ pb.NewTuple({data3, data3}),
+ pb.NewTuple({data3, data4}),
+ pb.NewTuple({data3, data5}),
+
+ pb.NewTuple({data4, data1}),
+ pb.NewTuple({data4, data2}),
+ pb.NewTuple({data4, data3}),
+ pb.NewTuple({data4, data4}),
+ pb.NewTuple({data4, data5}),
+
+ pb.NewTuple({data5, data1}),
+ pb.NewTuple({data5, data2}),
+ pb.NewTuple({data5, data3}),
+ pb.NewTuple({data5, data4}),
+ pb.NewTuple({data5, data5}),
+ });
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({
+ pb.AggrEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrNotEquals(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrLess(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrLessOrEqual(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrGreater(pb.Nth(item, 0), pb.Nth(item, 1)),
+ pb.AggrGreaterOrEqual(pb.Nth(item, 0), pb.Nth(item, 1))
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(!item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(!item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).template Get<bool>()); // ==
+ UNIT_ASSERT(!item.GetElement(1).template Get<bool>()); // !=
+ UNIT_ASSERT(!item.GetElement(2).template Get<bool>()); // <
+ UNIT_ASSERT(item.GetElement(3).template Get<bool>()); // <=
+ UNIT_ASSERT(!item.GetElement(4).template Get<bool>()); // >
+ UNIT_ASSERT(item.GetElement(5).template Get<bool>()); // >=
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestIncDec) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 4, 1);
+ const auto data2 = pb.NewDecimalLiteral(-9999, 4, 1);
+ const auto data3 = pb.NewDecimalLiteral(-7, 4, 1);
+ const auto data4 = pb.NewDecimalLiteral(0, 4, 1);
+ const auto data5 = pb.NewDecimalLiteral(13, 4, 1);
+ const auto data6 = pb.NewDecimalLiteral(9999, 4, 1);
+ const auto data7 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 4, 1);
+ const auto data8 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 4, 1);
+
+ const auto list = pb.NewList(pb.NewDecimalType(4, 1), {data1, data2, data3, data4, data5, data6, data7, data8});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Increment(item), pb.Decrement(item)});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -9998);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -6);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -8);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == +1);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == -1);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 14);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 12);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 9998);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == +NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinusAbs) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDecimalLiteral(-NYql::NDecimal::Nan(), 10, 1);
+ const auto data1 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 1);
+ const auto data3 = pb.NewDecimalLiteral(-7, 10, 1);
+ const auto data4 = pb.NewDecimalLiteral(0, 10, 1);
+ const auto data5 = pb.NewDecimalLiteral(13, 10, 1);
+ const auto data7 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 1);
+ const auto data8 = pb.NewDecimalLiteral(+NYql::NDecimal::Nan(), 10, 1);
+
+ const auto list = pb.NewList(pb.NewDecimalType(10, 1), {data0, data1, data3, data4, data5, data7, data8});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Minus(item), pb.Abs(item)});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 7);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 7);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == 0);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == 0);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -13);
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +13);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == +NYql::NDecimal::Inf());
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0).GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(item.GetElement(1).GetInt128() == NYql::NDecimal::Nan());
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFromString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("0.0");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("NAN");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("1.0");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-.1");
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("3.1415926");
+ const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("+inf");
+ const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-INF");
+ const auto data7 = pb.NewDataLiteral<NUdf::EDataSlot::String>(".123E+2");
+ const auto data8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("56.78e-3");
+ const auto type = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6, data7, data8});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.StrictFromString(item, pb.NewDecimalType(10, 7));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == NYql::NDecimal::Nan());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 10000000);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -1000000);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 31415926);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == +NYql::NDecimal::Inf());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == -NYql::NDecimal::Inf());
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 123000000);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetInt128() == 567800);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestToString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDecimalLiteral(0, 10, 7);
+ const auto data1 = pb.NewDecimalLiteral(NYql::NDecimal::Nan(), 10, 7);
+ const auto data2 = pb.NewDecimalLiteral(10000000, 10, 7);
+ const auto data3 = pb.NewDecimalLiteral(-1000000, 10, 7);
+ const auto data4 = pb.NewDecimalLiteral(31415926, 10, 7);
+ const auto data5 = pb.NewDecimalLiteral(+NYql::NDecimal::Inf(), 10, 7);
+ const auto data6 = pb.NewDecimalLiteral(-NYql::NDecimal::Inf(), 10, 7);
+ const auto type = pb.NewDecimalType(10, 7);
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4, data5, data6});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.ToString(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "0");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "nan");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "1");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "-0.1");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "3.1415926");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "inf");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "-inf");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFromStringToDouble) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("0");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("+3.332873");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-3.332873");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("+3.1415926");
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("-3.1415926");
+
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Convert(pb.FromString(item, pb.NewDecimalType(35,25)), pb.NewDataType(NUdf::TDataType<double>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<double>(), 0.);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<double>(), +3.332873);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<double>(), -3.332873);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<double>(), +3.1415926);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<double>(), -3.1415926);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFromUtf8ToFloat) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("0");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("+24.75");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("-24.75");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("+42.42");
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("-42.42");
+
+ const auto list = pb.NewList(type, {data0, data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.Convert(pb.FromString(item, pb.NewDecimalType(35,25)), pb.NewDataType(NUdf::TDataType<float>::Id));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<float>(), 0.f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<float>(), +24.75f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<float>(), -24.75f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<float>(), +42.42f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<float>(), -42.42f);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp
new file mode 100644
index 00000000000..ac6d4c356b2
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp
@@ -0,0 +1,393 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLDictRelatedNodesTest) {
+ Y_UNIT_TEST_LLVM(TestDictLength) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1);
+ const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto key3 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto payload1 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ TVector<std::pair<TRuntimeNode, TRuntimeNode>> dictItems;
+ dictItems.push_back(std::make_pair(key1, payload1));
+ dictItems.push_back(std::make_pair(key2, payload2));
+ dictItems.push_back(std::make_pair(key3, payload3));
+ const auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id),
+ pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false);
+ const auto dict = pgmBuilder.NewDict(dictType, dictItems);
+ const auto pgmReturn = pgmBuilder.Length(dict);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().template Get<ui64>(), 2);
+ }
+
+ Y_UNIT_TEST_LLVM(TestDictContains) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1);
+ const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto key3 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto missingKey = pgmBuilder.NewDataLiteral<ui32>(3);
+ const auto payload1 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ TVector<std::pair<TRuntimeNode, TRuntimeNode>> dictItems;
+ dictItems.push_back(std::make_pair(key1, payload1));
+ dictItems.push_back(std::make_pair(key2, payload2));
+ dictItems.push_back(std::make_pair(key3, payload3));
+ const auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id),
+ pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false);
+ const auto dict = pgmBuilder.NewDict(dictType, dictItems);
+ const auto keys = pgmBuilder.NewList(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), {key1, key2, missingKey});
+
+ const auto pgmReturn = pgmBuilder.Map(keys,
+ [&](TRuntimeNode key) {
+ return pgmBuilder.Contains(dict, key);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDictLookup) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1);
+ const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto key3 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto missingKey = pgmBuilder.NewDataLiteral<ui32>(3);
+ const auto payload1 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ TVector<std::pair<TRuntimeNode, TRuntimeNode>> dictItems;
+ dictItems.push_back(std::make_pair(key1, payload1));
+ dictItems.push_back(std::make_pair(key2, payload2));
+ dictItems.push_back(std::make_pair(key3, payload3));
+ const auto dictType = pgmBuilder.NewDictType(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id),
+ pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id), false);
+ const auto dict = pgmBuilder.NewDict(dictType, dictItems);
+ const auto keys = pgmBuilder.NewList(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), {key1, key2, missingKey});
+
+ const auto pgmReturn = pgmBuilder.Map(keys,
+ [&](TRuntimeNode key) {
+ return pgmBuilder.Lookup(dict, key);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ UNBOXED_VALUE_STR_EQUAL(item, "A");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ UNBOXED_VALUE_STR_EQUAL(item, "B");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ template<bool Multi>
+ TRuntimeNode PrepareTestDict(TProgramBuilder& pgmBuilder, TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi,
+ const TProgramBuilder::TUnaryLambda& keySelector,
+ const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) {
+ const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1);
+ const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto key3 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto key4 = pgmBuilder.NewDataLiteral<ui32>(5);
+ const auto key5 = pgmBuilder.NewDataLiteral<ui32>(7);
+ const auto payload1 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("D");
+ const auto payload5 = pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("E");
+ auto structType = pgmBuilder.NewStructType(pgmBuilder.NewEmptyStructType(), "Key", pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ structType = pgmBuilder.NewStructType(structType, "Payload", pgmBuilder.NewDataType(NUdf::TDataType<char*>::Id));
+ const auto list = pgmBuilder.NewList(structType, {
+ pgmBuilder.AddMember(pgmBuilder.AddMember(pgmBuilder.NewEmptyStruct(), "Key", key3), "Payload", payload3),
+ pgmBuilder.AddMember(pgmBuilder.AddMember(pgmBuilder.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pgmBuilder.AddMember(pgmBuilder.AddMember(pgmBuilder.NewEmptyStruct(), "Key", key5), "Payload", payload5),
+ pgmBuilder.AddMember(pgmBuilder.AddMember(pgmBuilder.NewEmptyStruct(), "Key", key4), "Payload", payload4),
+ pgmBuilder.AddMember(pgmBuilder.AddMember(pgmBuilder.NewEmptyStruct(), "Key", key2), "Payload", payload2)
+ });
+ const auto dict = (pgmBuilder.*factory)(list, Multi,
+ [&](TRuntimeNode item) {
+ return pgmBuilder.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pgmBuilder.Member(item, "Payload");
+ }, false, 0);
+ return dict;
+ }
+
+ template<bool LLVM>
+ void TestConvertedDictContains(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi,
+ const TProgramBuilder::TUnaryLambda& keySelector,
+ const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) {
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+ const auto dict = PrepareTestDict<false>(pgmBuilder, factory);
+
+ const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1);
+ const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto missingKey = pgmBuilder.NewDataLiteral<ui32>(42);
+ const auto keys = pgmBuilder.NewList(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), {key1, key2, missingKey});
+
+ const auto pgmReturn = pgmBuilder.Map(keys,
+ [&](TRuntimeNode key) {
+ return pgmBuilder.Contains(dict, key);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ template<bool LLVM>
+ void TestConvertedDictLookup(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi,
+ const TProgramBuilder::TUnaryLambda& keySelector,
+ const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) {
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+ const auto dict = PrepareTestDict<false>(pgmBuilder, factory);
+
+ const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1);
+ const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto missingKey = pgmBuilder.NewDataLiteral<ui32>(18);
+ const auto keys = pgmBuilder.NewList(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), {key1, key2, missingKey});
+
+ const auto pgmReturn = pgmBuilder.Map(keys,
+ [&](TRuntimeNode key) {
+ return pgmBuilder.Lookup(dict, key);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ UNBOXED_VALUE_STR_EQUAL(item, "A");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ UNBOXED_VALUE_STR_EQUAL(item, "C");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSortedDictContains) {
+ TestConvertedDictContains<LLVM>(&TProgramBuilder::ToSortedDict);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSortedDictLookup) {
+ TestConvertedDictLookup<LLVM>(&TProgramBuilder::ToSortedDict);
+ }
+
+ Y_UNIT_TEST_LLVM(TestHashedDictContains) {
+ TestConvertedDictContains<LLVM>(&TProgramBuilder::ToHashedDict);
+ }
+
+ Y_UNIT_TEST_LLVM(TestHashedDictLookup) {
+ TestConvertedDictLookup<LLVM>(&TProgramBuilder::ToHashedDict);
+ }
+
+ template<bool LLVM, bool SortBeforeCompare>
+ void TestDictItemsImpl(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi,
+ const TProgramBuilder::TUnaryLambda& keySelector,
+ const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+ const auto dict = PrepareTestDict<false>(pgmBuilder, factory);
+ const auto pgmReturn = pgmBuilder.DictItems(dict);
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ std::vector<std::pair<ui32, TString>> items;
+ for (NUdf::TUnboxedValue item; iterator.Next(item);) {
+ const auto& pay = item.GetElement(1);
+ items.emplace_back(item.GetElement(0).template Get<ui32>(), pay.AsStringRef());
+ }
+
+ if (SortBeforeCompare) {
+ std::sort(items.begin(), items.end(), [](const std::pair<ui32, TString>& left, const std::pair<ui32, TString>& right) {
+ return left.first < right.first;
+ });
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(items.size(), 4U);
+ UNIT_ASSERT_VALUES_EQUAL(items[0].first, 1);
+ UNIT_ASSERT_VALUES_EQUAL(items[0].second, "A");
+ UNIT_ASSERT_VALUES_EQUAL(items[1].first, 2);
+ UNIT_ASSERT_VALUES_EQUAL(items[1].second, "C");
+ UNIT_ASSERT_VALUES_EQUAL(items[2].first, 5);
+ UNIT_ASSERT_VALUES_EQUAL(items[2].second, "D");
+ UNIT_ASSERT_VALUES_EQUAL(items[3].first, 7);
+ UNIT_ASSERT_VALUES_EQUAL(items[3].second, "E");
+ }
+
+ template<bool LLVM, bool SortBeforeCompare>
+ void TestDictKeysImpl(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi,
+ const TProgramBuilder::TUnaryLambda& keySelector,
+ const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+ const auto dict = PrepareTestDict<false>(pgmBuilder, factory);
+ const auto pgmReturn = pgmBuilder.DictKeys(dict);
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ std::vector<ui32> items;
+ for (NUdf::TUnboxedValue item; iterator.Next(item);) {
+ items.emplace_back(item.template Get<ui32>());
+ }
+
+ if (SortBeforeCompare) {
+ std::sort(items.begin(), items.end());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(items.size(), 4U);
+ UNIT_ASSERT_VALUES_EQUAL(items[0], 1);
+ UNIT_ASSERT_VALUES_EQUAL(items[1], 2);
+ UNIT_ASSERT_VALUES_EQUAL(items[2], 5);
+ UNIT_ASSERT_VALUES_EQUAL(items[3], 7);
+ }
+
+ template<bool LLVM, bool SortBeforeCompare>
+ void TestDictPayloadsImpl(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi,
+ const TProgramBuilder::TUnaryLambda& keySelector,
+ const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+ const auto dict = PrepareTestDict<false>(pgmBuilder, factory);
+ const auto pgmReturn = pgmBuilder.DictPayloads(dict);
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ std::vector<TString> items;
+ for (NUdf::TUnboxedValue item; iterator.Next(item);) {
+ items.emplace_back(item.AsStringRef());
+ }
+
+ if (SortBeforeCompare) {
+ std::sort(items.begin(), items.end());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(items.size(), 4U);
+ UNIT_ASSERT_VALUES_EQUAL(items[0], "A");
+ UNIT_ASSERT_VALUES_EQUAL(items[1], "C");
+ UNIT_ASSERT_VALUES_EQUAL(items[2], "D");
+ UNIT_ASSERT_VALUES_EQUAL(items[3], "E");
+ }
+
+ Y_UNIT_TEST_LLVM(TestSortedDictItems) {
+ TestDictItemsImpl<LLVM, false>(&TProgramBuilder::ToSortedDict);
+ }
+
+ Y_UNIT_TEST_LLVM(TestHashedDictItems) {
+ TestDictItemsImpl<LLVM, true>(&TProgramBuilder::ToHashedDict);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSortedDictKeys) {
+ TestDictKeysImpl<LLVM, false>(&TProgramBuilder::ToSortedDict);
+ }
+
+ Y_UNIT_TEST_LLVM(TestHashedDictKeys) {
+ TestDictKeysImpl<LLVM, true>(&TProgramBuilder::ToHashedDict);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSortedPayloadsKeys) {
+ TestDictPayloadsImpl<LLVM, false>(&TProgramBuilder::ToSortedDict);
+ }
+
+ Y_UNIT_TEST_LLVM(TestHashedPayloadsKeys) {
+ TestDictPayloadsImpl<LLVM, true>(&TProgramBuilder::ToHashedDict);
+ }
+
+ template<bool LLVM>
+ void TestConvertedMultiDictLookup(TRuntimeNode(TProgramBuilder::* factory)(TRuntimeNode list, bool multi,
+ const TProgramBuilder::TUnaryLambda& keySelector,
+ const TProgramBuilder::TUnaryLambda& payloadSelector, bool isCompact, ui64 itemsCountHint)) {
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+ const auto dict = PrepareTestDict<true>(pgmBuilder, factory);
+ const auto key1 = pgmBuilder.NewDataLiteral<ui32>(1);
+ const auto key2 = pgmBuilder.NewDataLiteral<ui32>(2);
+ const auto missingKey = pgmBuilder.NewDataLiteral<ui32>(3);
+ const auto keys = pgmBuilder.NewList(pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id), {key1, key2, missingKey});
+ const auto pgmReturn = pgmBuilder.Map(keys,
+ [&](TRuntimeNode key) {
+ return pgmBuilder.Lookup(dict, key);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item, item2;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ auto iter2 = item.GetListIterator();
+ UNIT_ASSERT(iter2.Next(item2));
+ UNBOXED_VALUE_STR_EQUAL(item2, "A");
+ UNIT_ASSERT(!iter2.Next(item2));
+ UNIT_ASSERT(!iter2.Next(item2));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ iter2 = item.GetListIterator();
+ UNIT_ASSERT(iter2.Next(item2));
+ UNBOXED_VALUE_STR_EQUAL(item2, "C");
+ UNIT_ASSERT(iter2.Next(item2));
+ UNBOXED_VALUE_STR_EQUAL(item2, "B");
+ UNIT_ASSERT(!iter2.Next(item2));
+ UNIT_ASSERT(!iter2.Next(item2));
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSortedMultiDictLookup) {
+ TestConvertedMultiDictLookup<LLVM>(&TProgramBuilder::ToSortedDict);
+ }
+
+ Y_UNIT_TEST_LLVM(TestHashedMultiDictLookup) {
+ TestConvertedMultiDictLookup<LLVM>(&TProgramBuilder::ToHashedDict);
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp
new file mode 100644
index 00000000000..e2c1c4dc690
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_filters_ut.cpp
@@ -0,0 +1,1120 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLFiltersTest) {
+ Y_UNIT_TEST_LLVM(TestSkipNullMembers) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}});
+
+ const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(1))}, {"Payload", pb.NewEmptyOptional(dataType)}});
+ const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewDataLiteral<i32>(2))}});
+ const auto data3 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(3))}, {"Payload", pb.NewEmptyOptional(dataType)}});
+
+ const auto list = pb.NewList(structType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.SkipNullMembers(list, {"Payload"});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterNullMembers) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}});
+
+ const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(1))}, {"Payload", pb.NewEmptyOptional(dataType)}});
+ const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewDataLiteral<i32>(2))}});
+ const auto data3 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(3))}, {"Payload", pb.NewEmptyOptional(dataType)}});
+
+ const auto list = pb.NewList(structType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.FilterNullMembers(list, {"Payload"});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterNullMembersMultiOptional) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)));
+ const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}});
+ const auto justNothing = pb.NewOptional(pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i32>::Id));
+
+ const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(1)))}, {"Payload", pb.NewEmptyOptional(dataType)}});
+ const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(2)))}});
+ const auto data3 = pb.NewStruct(structType, {{"Key", justNothing}, {"Payload", justNothing}});
+
+ const auto list = pb.NewList(structType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.FilterNullMembers(list, {"Payload"});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipNullMembersOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}});
+
+ const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(1))}, {"Payload", pb.NewEmptyOptional(dataType)}});
+ const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewDataLiteral<i32>(2))}});
+ const auto data3 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(3))}, {"Payload", pb.NewEmptyOptional(dataType)}});
+
+ const auto list = pb.NewList(structType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.SkipNullMembers(pb.ToFlow(list), {"Payload"}));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterNullMembersOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}});
+
+ const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(1))}, {"Payload", pb.NewEmptyOptional(dataType)}});
+ const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewDataLiteral<i32>(2))}});
+ const auto data3 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewDataLiteral<i32>(3))}, {"Payload", pb.NewEmptyOptional(dataType)}});
+
+ const auto list = pb.NewList(structType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.FilterNullMembers(pb.ToFlow(list), {"Payload"}));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterNullMembersMultiOptionalOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)));
+ const auto structType = pb.NewStructType({{"Key", dataType}, {"Payload", dataType}});
+ const auto justNothing = pb.NewOptional(pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i32>::Id));
+
+ const auto data1 = pb.NewStruct(structType, {{"Key", pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(1)))}, {"Payload", pb.NewEmptyOptional(dataType)}});
+ const auto data2 = pb.NewStruct(structType, {{"Key", pb.NewEmptyOptional(dataType)}, {"Payload", pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(2)))}});
+ const auto data3 = pb.NewStruct(structType, {{"Key", justNothing}, {"Payload", justNothing}});
+
+ const auto list = pb.NewList(structType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.FilterNullMembers(pb.ToFlow(list), {"Payload"}));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipNullElements) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.SkipNullElements(list, {1U, 2U});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterNullElements) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.FilterNullElements(list, {1U, 2U});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterNullElementsMultiOptional) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+ const auto justNothing = pb.NewOptional(pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i32>::Id));
+
+ const auto data1 = pb.NewTuple(tupleType, {justNothing, pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-1)))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(2))), justNothing});
+ const auto data3 = pb.NewTuple(tupleType, {justNothing, pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-3)))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(4))), justNothing, pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-4)))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.FilterNullElements(list, {1U, 2U});
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipNullElementsOverOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.FromFlow(pb.SkipNullElements(pb.ToFlow(list), {1U, 2U}));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterNullElementsOverOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.FromFlow(pb.FilterNullElements(pb.ToFlow(list), {1U, 2U}));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterNullElementsOverMultiOptionalOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id)));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+ const auto justNothing = pb.NewOptional(pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<i32>::Id));
+
+ const auto data1 = pb.NewTuple(tupleType, {justNothing, pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-1)))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(2))), justNothing});
+ const auto data3 = pb.NewTuple(tupleType, {justNothing, pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-3)))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(4))), justNothing, pb.NewOptional(pb.NewOptional(pb.NewDataLiteral<i32>(-4)))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.FromFlow(pb.FilterNullElements(pb.ToFlow(list), {1U, 2U}));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -4);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterOverList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto optionalType = pb.NewOptionalType(dataType);
+ const auto list = pb.NewList(optionalType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(data2)});
+
+ const auto pgmReturn = pb.Filter(list,
+ [&](TRuntimeNode item) {
+ return pb.Exists(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterOverStream) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui32>(0);
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto data3 = pb.NewDataLiteral<ui32>(3);
+ const auto data4 = pb.NewDataLiteral<ui32>(4);
+ const auto data5 = pb.NewDataLiteral<ui32>(5);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5});
+ const auto pgmReturn = pb.Filter(pb.Iterator(list, {}),
+ [&](TRuntimeNode item) {
+ return pb.Greater(pb.Unwrap(pb.Mod(item, data3), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), data1);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 5);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui32>(0);
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto data3 = pb.NewDataLiteral<ui32>(3);
+ const auto data4 = pb.NewDataLiteral<ui32>(4);
+ const auto data5 = pb.NewDataLiteral<ui32>(5);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3, data4, data5});
+ const auto pgmReturn = pb.FromFlow(pb.Filter(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return pb.Greater(pb.Unwrap(pb.Mod(item, data3), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), data1);
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 5);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterOverListLazy) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ const auto data = pb.NewEmptyOptional(dataType);
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2U));
+ const auto list = pb.NewList(dataType, {data, data2});
+
+ const auto pgmReturn = pb.Filter(pb.LazyList(list),
+ [&](TRuntimeNode item) { return pb.Exists(item); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(item);
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterByString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto optionalType = pb.NewOptionalType(dataType);
+ const auto data0 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("000"));
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("100"));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("200"));
+ const auto data3 = pb.NewEmptyOptional(optionalType);
+ const auto list = pb.NewList(optionalType, {data0, data1, data2, data3, data1, data0});
+
+ const auto pgmReturn = pb.Filter(list,
+ [&](TRuntimeNode item) {
+ return pb.Coalesce(pb.Equals(item, data1), pb.NewDataLiteral<bool>(false));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), 2ULL);
+ UNBOXED_VALUE_STR_EQUAL(result.GetElement(0U), "100");
+ UNBOXED_VALUE_STR_EQUAL(result.GetElement(1U), "100");
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipWhile) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.SkipWhile(list,
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, data2);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "200");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "300");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestTakeWhile) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.TakeWhile(list,
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, data2);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "000");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "100");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipWhileOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.SkipWhile(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, data2);
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "200");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "300");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestTakeWhileOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.TakeWhile(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, data2);
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "000");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "100");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipWhileInclusive) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.SkipWhileInclusive(list,
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, data2);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "300");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestTakeWhileInclusive) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.TakeWhileInclusive(list,
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, data2);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "000");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "100");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "200");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipWhileInclusiveOnEmptyList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {});
+
+ const auto pgmReturn = pb.SkipWhileInclusive(list,
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, pb.NewDataLiteral<NUdf::EDataSlot::String>("000"));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestTakeWhileInclusiveOnEmptyList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {});
+
+ const auto pgmReturn = pb.TakeWhileInclusive(list,
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, pb.NewDataLiteral<NUdf::EDataSlot::String>("000"));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipWhileInclusiveOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.SkipWhileInclusive(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, data2);
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "300");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestTakeWhileInclusiveOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.TakeWhileInclusive(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, data2);
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "000");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "100");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "200");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDateToStringCompleteCheck) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(NUdf::MAX_DATE), pb.NewDataLiteral<ui16>(1U));
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+ const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list,
+ [&](TRuntimeNode item) {
+ const auto date = pb.ToIntegral(item, dateType);
+ const auto utf8 = pb.ToString<true>(date);
+ return pb.AggrNotEquals(date, pb.FromString(utf8, dateType));
+ }
+ )));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNIT_ASSERT(result.template Get<bool>());
+ }
+
+ Y_UNIT_TEST_LLVM(TestTzDateToStringCompleteCheck) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(NUdf::MAX_DATE), pb.NewDataLiteral<ui16>(1U));
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+ const auto dateTypeTz = pb.NewDataType(NUdf::EDataSlot::TzDate, true);
+ const auto canada = pb.NewDataLiteral<ui16>(375U);
+ const auto europe = pb.NewDataLiteral<ui16>(459U);
+ const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list,
+ [&](TRuntimeNode item) {
+ const auto date = pb.Unwrap(pb.ToIntegral(item, dateType), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto date1 = pb.Unwrap(pb.FromString(pb.ToString<true>(pb.AddTimezone(date, canada)), dateTypeTz), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto date2 = pb.Unwrap(pb.FromString(pb.ToString<true>(pb.AddTimezone(date, europe)), dateTypeTz), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ return pb.Or({pb.NotEquals(date, date1), pb.NotEquals(date, date2)});
+ }
+ )));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNIT_ASSERT(result.template Get<bool>());
+ }
+
+ Y_UNIT_TEST_LLVM(TestInt16ToFloatCompleteCheck) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.ListFromRange(pb.NewDataLiteral<i16>(std::numeric_limits<i16>::min()), pb.NewDataLiteral<i16>(std::numeric_limits<i16>::max()), pb.NewDataLiteral<i16>(1));
+ const auto type = pb.NewDataType(NUdf::EDataSlot::Float);
+ const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list,
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, pb.Convert(item, type));
+ }
+ )));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNIT_ASSERT(result.template Get<bool>());
+ }
+
+ Y_UNIT_TEST_LLVM(TestUint16ToFloatCompleteCheck) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(std::numeric_limits<ui16>::min()), pb.NewDataLiteral<ui16>(std::numeric_limits<ui16>::max()), pb.NewDataLiteral<ui16>(1U));
+ const auto type = pb.NewDataType(NUdf::EDataSlot::Float);
+ const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list,
+ [&](TRuntimeNode item) {
+ return pb.NotEquals(item, pb.Convert(item, type));
+ }
+ )));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNIT_ASSERT(result.template Get<bool>());
+ }
+
+ Y_UNIT_TEST_LLVM(TestDateToDatetimeCompleteCheck) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(NUdf::MAX_DATE), pb.NewDataLiteral<ui16>(1U));
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+ const auto datetimeType = pb.NewDataType(NUdf::EDataSlot::Datetime, true);
+ const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list,
+ [&](TRuntimeNode item) {
+ const auto date = pb.ToIntegral(item, dateType);
+ return pb.Coalesce(pb.NotEquals(date, pb.Convert(date, datetimeType)), pb.NewDataLiteral(false));
+ }
+ )));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNIT_ASSERT(result.template Get<bool>());
+ }
+
+ Y_UNIT_TEST_LLVM(TestTzDateToDatetimeCompleteCheck) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(NUdf::MAX_DATE), pb.NewDataLiteral<ui16>(1U));
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+ const auto datetimeType = pb.NewDataType(NUdf::EDataSlot::Datetime);
+ const auto canada = pb.NewDataLiteral<ui16>(375U);
+ const auto europe = pb.NewDataLiteral<ui16>(459U);
+ const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list,
+ [&](TRuntimeNode item) {
+ const auto date = pb.ToIntegral(item, dateType);
+ const auto date1 = pb.Unwrap(pb.AddTimezone(date, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto date2 = pb.Unwrap(pb.AddTimezone(date, europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ return pb.Or({pb.NotEquals(date1, pb.Cast(date1, datetimeType)), pb.NotEquals(date2, pb.Cast(date2, datetimeType))});
+ }
+ )));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNIT_ASSERT(result.template Get<bool>());
+ }
+
+ Y_UNIT_TEST_LLVM(TestDateAddTimezoneAndCastOrderCompleteCheck) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.ListFromRange(pb.NewDataLiteral<ui16>(0U), pb.NewDataLiteral<ui16>(NUdf::MAX_DATE), pb.NewDataLiteral<ui16>(1U));
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+ const auto datetimeType = pb.NewDataType(NUdf::EDataSlot::Datetime);
+ const auto datetimeTypeTz = pb.NewDataType(NUdf::EDataSlot::TzDatetime);
+ const auto canada = pb.NewDataLiteral<ui16>(375U);
+ const auto europe = pb.NewDataLiteral<ui16>(459U);
+ const auto pgmReturn = pb.Not(pb.HasItems(pb.Filter(list,
+ [&](TRuntimeNode item) {
+ const auto date = pb.Unwrap(pb.ToIntegral(item, dateType), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto date1_1 = pb.Cast(pb.Unwrap(pb.AddTimezone(date, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), datetimeTypeTz);
+ const auto date1_2 = pb.Unwrap(pb.AddTimezone(pb.Cast(date, datetimeType), canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto date2_1 = pb.Cast(pb.Unwrap(pb.AddTimezone(date, europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), datetimeTypeTz);
+ const auto date2_2 = pb.Unwrap(pb.AddTimezone(pb.Cast(date, datetimeType), europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ return pb.Or({pb.NotEquals(date1_1, date1_2), pb.NotEquals(date2_1, date2_2), pb.NotEquals(date1_1, date2_2), pb.NotEquals(date2_1, date1_2)});
+ }
+ )));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto result = graph->GetValue();
+ UNIT_ASSERT(result.template Get<bool>());
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterWithLimitOverList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui32>(0);
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto data3 = pb.NewDataLiteral<ui32>(3);
+ const auto data4 = pb.NewDataLiteral<ui32>(4);
+ const auto data5 = pb.NewDataLiteral<ui32>(5);
+ const auto data6 = pb.NewDataLiteral<ui32>(6);
+ const auto limit = pb.NewDataLiteral<ui64>(3ULL);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto optionalType = pb.NewOptionalType(dataType);
+ const auto list = pb.NewList(optionalType, {
+ pb.NewOptional(data0),
+ pb.NewOptional(data1),
+ pb.NewOptional(data2),
+ pb.NewOptional(data3),
+ pb.NewOptional(data4),
+ pb.NewOptional(data5),
+ pb.NewOptional(data3),
+ pb.NewOptional(data4),
+ pb.NewOptional(data6),
+ pb.NewEmptyOptional(optionalType),
+ pb.NewEmptyOptional(optionalType)
+ });
+
+ const auto pgmReturn = pb.Filter(list, limit,
+ [&](TRuntimeNode item) {
+ return pb.AggrEquals(pb.Unwrap(pb.Mod(item, data2), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0), data0);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterWithLimitOverEmptyList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto limit = pb.NewDataLiteral<ui64>(3ULL);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto optionalType = pb.NewOptionalType(dataType);
+ const auto list = pb.NewEmptyList(optionalType);
+
+ const auto pgmReturn = pb.Filter(list, limit,
+ [&](TRuntimeNode) {
+ return pb.NewDataLiteral<bool>(true);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterWithLimitOverLazyList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto start = pb.NewDataLiteral<ui64>(0ULL);
+ const auto stop = pb.NewDataLiteral<ui64>(1000000000000ULL);
+ const auto step = pb.NewDataLiteral<ui64>(1ULL);
+
+ const auto limit = pb.NewDataLiteral<ui64>(7ULL);
+ const auto list = pb.ListFromRange(start, stop, step);
+
+ const auto pgmReturn = pb.Filter(list, limit,
+ [&](TRuntimeNode item) {
+ return pb.AggrEquals(pb.CountBits(item), pb.NewDataLiteral<ui64>(3ULL));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 7ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 11ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 13ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 14ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 19ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 21ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 22ULL);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterWithSmallLimitGetTailOfLargeList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto start = pb.NewDataLiteral<ui64>(0ULL);
+ const auto stop = pb.NewDataLiteral<ui64>(100100LL);
+ const auto step = pb.NewDataLiteral<ui64>(1ULL);
+
+ const auto limit = pb.NewDataLiteral<ui64>(3ULL);
+ const auto list = pb.ListFromRange(start, stop, step);
+
+ const auto pgmReturn = pb.Filter(pb.Collect(list), limit,
+ [&](TRuntimeNode item) {
+ return pb.AggrGreaterOrEqual(item, pb.NewDataLiteral<ui64>(100000LL));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 100000LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 100001LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 100002LL);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterWithLimitOverEnumerate) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto start = pb.NewDataLiteral<i64>(0LL);
+ const auto stop = pb.NewDataLiteral<i64>(-1000000000000LL);
+ const auto step = pb.NewDataLiteral<i64>(-1LL);
+
+ const auto estart = pb.NewDataLiteral<ui64>(42ULL);
+ const auto estep = pb.NewDataLiteral<ui64>(3ULL);
+
+ const auto limit = pb.NewDataLiteral<ui64>(5ULL);
+ const auto list = pb.Enumerate(pb.ListFromRange(start, stop, step), estart, estep);
+
+ const auto pgmReturn = pb.Filter(list, limit,
+ [&](TRuntimeNode item) {
+ return pb.AggrEquals(pb.CountBits(pb.Add(pb.Nth(item, 0), pb.Nth(item, 1))), pb.NewDataLiteral<i64>(4LL));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui64>(), 48ULL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<i64>(), -2LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui64>(), 60ULL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<i64>(), -6LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui64>(), 66ULL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<i64>(), -8LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui64>(), 69ULL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<i64>(), -9LL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0U).template Get<ui64>(), 96ULL);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1U).template Get<i64>(), -18LL);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterWithLimitOverStream) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewStreamType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id)));
+ const auto stream = pb.ChainMap(pb.SourceOf(type), pb.NewDataLiteral<ui64>(0ULL),
+ [&](TRuntimeNode, TRuntimeNode state) -> TRuntimeNodePair {
+ return {state, pb.Increment(state)};
+ }
+ );
+
+ const auto limit = pb.NewDataLiteral<ui64>(7ULL);
+ const auto pgmReturn = pb.Filter(stream, limit,
+ [&](TRuntimeNode item) {
+ return pb.AggrEquals(pb.CountBits(item), pb.NewDataLiteral<ui64>(3ULL));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 7ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 11ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 13ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 14ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 19ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 21ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 22ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterWithLimitOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewFlowType(pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<ui32>::Id)));
+ const auto flow = pb.ChainMap(pb.SourceOf(type), pb.NewDataLiteral<ui64>(0ULL),
+ [&](TRuntimeNode, TRuntimeNode state) -> TRuntimeNodePair {
+ return {state, pb.Increment(state)};
+ }
+ );
+
+ const auto limit = pb.NewDataLiteral<ui64>(7ULL);
+ const auto pgmReturn = pb.FromFlow(pb.Filter(flow, limit,
+ [&](TRuntimeNode item) {
+ return pb.AggrEquals(pb.CountBits(item), pb.NewDataLiteral<ui64>(3ULL));
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 7ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 11ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 13ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 14ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 19ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 21ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 22ULL);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp
new file mode 100644
index 00000000000..8ad155473d5
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_flatmap_ut.cpp
@@ -0,0 +1,853 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLFlatMapTest) {
+ Y_UNIT_TEST_LLVM(TestOverListAndPartialLists) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2});
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode item) {
+ return pb.NewList(dataType, {pb.Add(item, data1), pb.Mul(item, data2)});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverListAndStreams) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<i8>(3);
+ const auto data2 = pb.NewDataLiteral<i8>(-7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i8>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2});
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode item) {
+ return pb.Iterator(pb.NewList(dataType, {pb.Plus(item), pb.Minus(item)}), {});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), -7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i8>(), 7);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverStreamAndPartialLists) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui16>(10);
+ const auto data2 = pb.NewDataLiteral<ui16>(20);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2});
+ const auto pgmReturn = pb.FlatMap(pb.Iterator(list, {}),
+ [&](TRuntimeNode item) {
+ return pb.NewList(dataType, {pb.Sub(item, data1), pb.Unwrap(pb.Div(item, data2), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 10);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverFlowAndPartialLists) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui16>(10);
+ const auto data2 = pb.NewDataLiteral<ui16>(20);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(pb.Iterator(list, {})),
+ [&](TRuntimeNode item) {
+ return pb.NewList(dataType, {pb.Sub(item, data1), pb.Unwrap(pb.Div(item, data2), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)});
+ }));
+
+ const auto& graph = setup.BuildGraph(pgmReturn);
+ const NUdf::TUnboxedValue& iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 10);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverStreamAndStreams) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i32>(-100);
+ const auto data0 = pb.NewDataLiteral<i32>(0);
+ const auto data1 = pb.NewDataLiteral<i32>(3);
+ const auto data2 = pb.NewDataLiteral<i32>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FlatMap(pb.Iterator(list, {}),
+ [&](TRuntimeNode item) {
+ return pb.Iterator(pb.NewList(pb.NewOptionalType(dataType),
+ {pb.Mod(data, item), pb.Div(data, item)}), {});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -33);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -14);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverFlowAndStreams) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i32>(-100);
+ const auto data0 = pb.NewDataLiteral<i32>(0);
+ const auto data1 = pb.NewDataLiteral<i32>(3);
+ const auto data2 = pb.NewDataLiteral<i32>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return pb.Iterator(pb.NewList(pb.NewOptionalType(dataType),
+ {pb.Mod(data, item), pb.Div(data, item)}), {});
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -33);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -14);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverFlowAndFlows) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i32>(-100);
+ const auto data0 = pb.NewDataLiteral<i32>(0);
+ const auto data1 = pb.NewDataLiteral<i32>(3);
+ const auto data2 = pb.NewDataLiteral<i32>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return pb.ToFlow(pb.NewList(pb.NewOptionalType(dataType),
+ {pb.Mod(data, item), pb.Div(data, item)}));
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -33);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -14);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverListAndFlows) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i32>(-100);
+ const auto data0 = pb.NewDataLiteral<i32>(0);
+ const auto data1 = pb.NewDataLiteral<i32>(3);
+ const auto data2 = pb.NewDataLiteral<i32>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.FlatMap(list,
+ [&](TRuntimeNode item) {
+ return pb.ToFlow(pb.NewList(pb.NewOptionalType(dataType),
+ {pb.Mod(data, item), pb.Div(data, item)}));
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -33);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -14);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverFlowAndIndependentFlows) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i32>(-100);
+ const auto data0 = pb.NewDataLiteral<i32>(0);
+ const auto data1 = pb.NewDataLiteral<i32>(3);
+ const auto data2 = pb.NewDataLiteral<i32>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(list),
+ [&](TRuntimeNode) {
+ return pb.Map(pb.ToFlow(pb.NewList(dataType, {data, data})), [&](TRuntimeNode it) { return pb.Abs(it); });
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverListAndIndependentFlows) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i32>(-100);
+ const auto data0 = pb.NewDataLiteral<i32>(0);
+ const auto data1 = pb.NewDataLiteral<i32>(3);
+ const auto data2 = pb.NewDataLiteral<i32>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.FlatMap(list,
+ [&](TRuntimeNode) {
+ return pb.Map(pb.ToFlow(pb.NewList(dataType, {data, data})), [&](TRuntimeNode it) { return pb.Minus(it); });
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverFlowAndPartialOptionals) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i64>(-100);
+ const auto data0 = pb.NewDataLiteral<i64>(0);
+ const auto data1 = pb.NewDataLiteral<i64>(3);
+ const auto data2 = pb.NewDataLiteral<i64>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i64>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(pb.Iterator(list, {})),
+ [&](TRuntimeNode item) {
+ return pb.Div(data, item);
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -33);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -14);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverStreamAndPartialOptionals) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i64>(-100);
+ const auto data0 = pb.NewDataLiteral<i64>(0);
+ const auto data1 = pb.NewDataLiteral<i64>(3);
+ const auto data2 = pb.NewDataLiteral<i64>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i64>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FlatMap(pb.Iterator(list, {}),
+ [&](TRuntimeNode item) {
+ return pb.Div(data, item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -33);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i64>(), -14);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverListAndPartialOptionals) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui32>(0);
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode item) {
+ return pb.Div(data2, item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverListAndDoubleOptionals) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui32>(0);
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode item) {
+ return pb.NewOptional(pb.Div(data2, item));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverOptionalAndPartialOptionals) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto list = pb.NewOptional(data2);
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode item) {
+ return pb.Div(item, data2);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT(value);
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 1);
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverOptionalAndPartialLists) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto list = pb.NewOptional(data2);
+ const auto pgmReturn = pb.FlatMap(list,
+ [&](TRuntimeNode item) {
+ return pb.Append(pb.AsList(item), data1);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverListAndPartialListsLazy) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1U);
+ const auto data2 = pb.NewDataLiteral<ui32>(2U);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2});
+
+ const auto pgmReturn = pb.FlatMap(pb.LazyList(list),
+ [&](TRuntimeNode item) {
+ return pb.NewList(dataType, {pb.Add(item, data1), pb.Mul(item, data2)});
+ });
+
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverListAndPartialOptionalsLazy) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<ui32>(0U);
+ const auto data2 = pb.NewDataLiteral<ui32>(2U);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data2});
+
+ const auto pgmReturn = pb.FlatMap(pb.LazyList(list),
+ [&](TRuntimeNode item) { return pb.Div(data2, item); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u
+ Y_UNIT_TEST_LLVM(TestNarrowWithList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowFlatMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U),pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.FlatMap(pb.NewList(dataType, items), [](TRuntimeNode item){ return item; }); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -3);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestNarrowWithFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowFlatMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U),pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.FlatMap(pb.ToFlow(pb.NewList(dataType, items)), [](TRuntimeNode item){ return item; }); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -3);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestNarrowWithIndependentFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+
+ const auto list = pb.NewList(tupleType, {data, data, data});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowFlatMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U),pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList) { return pb.Map(
+ pb.ToFlow(pb.NewList(pb.NewDataType(NUdf::TDataType<float>::Id), {pb.NewDataLiteral<float>(+1.f), pb.NewDataLiteral<float>(-1.f)})),
+ [&](TRuntimeNode item) { return pb.Minus(item); }); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -1.f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), +1.f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -1.f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), +1.f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), -1.f);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<float>(), +1.f);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestThinNarrowWithList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto tupleType = pb.NewTupleType({});
+
+ const auto data = pb.NewTuple(tupleType, {});
+ const auto list = pb.NewList(tupleType, {data, data, data});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowFlatMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode) -> TRuntimeNode::TList { return {}; }),
+ [&](TRuntimeNode::TList) -> TRuntimeNode { return pb.Replicate(pb.NewDataLiteral<i32>(7), pb.NewDataLiteral<ui64>(3), __FILE__, __LINE__, 0); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 7);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverFlowAndWideFlows) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i32>(-100);
+ const auto data0 = pb.NewDataLiteral<i32>(0);
+ const auto data1 = pb.NewDataLiteral<i32>(3);
+ const auto data2 = pb.NewDataLiteral<i32>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.FlatMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return pb.ExpandMap(pb.ToFlow(pb.NewList(pb.NewOptionalType(dataType),
+ {pb.Mod(data, item), pb.Div(data, item)})),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Plus(item), pb.Minus(item)}; });
+ }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -33);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +33);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -14);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +14);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverListAndWideFlows) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i32>(-100);
+ const auto data0 = pb.NewDataLiteral<i32>(0);
+ const auto data1 = pb.NewDataLiteral<i32>(3);
+ const auto data2 = pb.NewDataLiteral<i32>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.FlatMap(list,
+ [&](TRuntimeNode item) {
+ return pb.ExpandMap(pb.ToFlow(pb.NewList(pb.NewOptionalType(dataType),
+ {pb.Mod(data, item), pb.Div(data, item)})),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Minus(item), pb.Plus(item)}; });
+ }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +33);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -33);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +14);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -14);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverFlowAndIndependentWideFlows) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i32>(-100);
+ const auto data0 = pb.NewDataLiteral<i32>(0);
+ const auto data1 = pb.NewDataLiteral<i32>(3);
+ const auto data2 = pb.NewDataLiteral<i32>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.FlatMap(pb.ToFlow(list),
+ [&](TRuntimeNode) {
+ return pb.ExpandMap(pb.ToFlow(pb.NewList(dataType, {data, data})),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Plus(item), pb.Minus(item)}; });
+ }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverListAndIndependentWideFlows) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data = pb.NewDataLiteral<i32>(-100);
+ const auto data0 = pb.NewDataLiteral<i32>(0);
+ const auto data1 = pb.NewDataLiteral<i32>(3);
+ const auto data2 = pb.NewDataLiteral<i32>(7);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2});
+ const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.FlatMap(list,
+ [&](TRuntimeNode) {
+ return pb.ExpandMap(pb.ToFlow(pb.NewList(dataType, {data, data})),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Minus(item), pb.Plus(item)}; });
+ }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), +100);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -100);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+#endif
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp
new file mode 100644
index 00000000000..951aa0158b5
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_fold_ut.cpp
@@ -0,0 +1,1207 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+#include <random>
+#include <ctime>
+#include <algorithm>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLFoldNodeTest) {
+ Y_UNIT_TEST_LLVM(TestFoldOverList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data1 = pb.NewDataLiteral<ui32>(1);
+ auto data2 = pb.NewDataLiteral<ui32>(2);
+ auto data3 = pb.NewDataLiteral<ui32>(3);
+ auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto list = pb.NewList(dataType, {data1, data2, data3});
+ auto pgmReturn = pb.Fold(list, pb.NewDataLiteral<ui32>(0),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Add(item, state);
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto res = graph->GetValue().template Get<ui32>();
+ UNIT_ASSERT_VALUES_EQUAL(res, 6);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFold1OverEmptyList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto list = pb.NewEmptyList(dataType);
+ auto data2 = pb.NewDataLiteral<ui32>(2);
+ auto pgmReturn = pb.Fold1(list, [&](TRuntimeNode item) {
+ return pb.Mul(item, data2);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Add(item, state);
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto value = graph->GetValue();
+ UNIT_ASSERT(!value);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFold1OverSingleElementList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto data1 = pb.NewDataLiteral<ui32>(1);
+ auto data2 = pb.NewDataLiteral<ui32>(2);
+ auto list = pb.NewList(dataType, {data1});
+ auto pgmReturn = pb.Fold1(list,
+ [&](TRuntimeNode item) {
+ return pb.Mul(item, data2);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Add(item, state);
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto value = graph->GetValue();
+ UNIT_ASSERT(value);
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 2);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFold1OverManyElementList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto data1 = pb.NewDataLiteral<ui32>(1);
+ auto data2 = pb.NewDataLiteral<ui32>(2);
+ auto list = pb.NewList(dataType, {data1, data2});
+ auto pgmReturn = pb.Fold1(list,
+ [&](TRuntimeNode item) {
+ return pb.Mul(item, data2);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Add(item, state);
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto value = graph->GetValue();
+ UNIT_ASSERT(value);
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<ui32>(), 4);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFoldWithAggrAdd) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<float>::Id));
+ auto data1 = pb.NewOptional(pb.NewDataLiteral<float>(1));
+ auto data2 = pb.NewOptional(pb.NewDataLiteral<float>(2));
+ auto data3 = pb.NewOptional(pb.NewDataLiteral<float>(3));
+ auto data4 = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<float>::Id);
+ auto data0 = pb.NewOptional(pb.NewDataLiteral<float>(42));
+ auto list = pb.NewList(dataType, {data4, data3, data2, data1});
+
+ auto pgmReturn = pb.Fold(list, data0,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrAdd(pb.Increment(item), pb.Decrement(state));
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto res = graph->GetValue().template Get<float>();
+ UNIT_ASSERT_VALUES_EQUAL(res, 47);
+ }
+
+ Y_UNIT_TEST_LLVM(TestNestedApply) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data1 = pb.NewDataLiteral<i32>(1);
+ auto data2 = pb.NewDataLiteral<i32>(2);
+ auto data3 = pb.NewDataLiteral<i32>(3);
+ auto dataType = pb.NewDataType(NUdf::TDataType<i32>::Id);
+ auto list = pb.NewList(dataType, {data1, data2, data3});
+
+ const auto callType = TCallableTypeBuilder(pb.GetTypeEnvironment(), "TEST", dataType).Add(dataType).Add(dataType).Build();
+
+ auto pgmReturn = pb.Fold(list, pb.NewDataLiteral<i32>(100),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Apply(pb.Callable(callType,
+ [&](const TArrayRef<const TRuntimeNode>& args) {
+ return pb.Sub(args[1], args[0]);
+ }), {item, state});
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto res = graph->GetValue().template Get<i32>();
+ UNIT_ASSERT_VALUES_EQUAL(res, 94);
+ }
+
+ Y_UNIT_TEST_LLVM(TestLogicalOpts) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto truth = pb.NewDataLiteral(true);
+ auto falsehood = pb.NewDataLiteral(false);
+ auto type = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ auto args = pb.NewList(type, {truth, falsehood});
+
+ auto optTruth = pb.NewOptional(truth);
+ auto optFalsehood = pb.NewOptional(falsehood);
+ auto empty = pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id);
+ auto optType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<bool>::Id));
+ auto opts = pb.NewList(optType, {empty, optTruth, optFalsehood});
+
+ auto pgmReturn = pb.Fold(opts, pb.NewEmptyList(optType),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ const auto append = pb.Append(state, pb.Not(item));
+
+ const auto one = pb.Fold(args, pb.NewEmptyList(optType),
+ [&](TRuntimeNode item2, TRuntimeNode state2) {
+ state2 = pb.Append(state2, pb.And({item, item2}));
+ state2 = pb.Append(state2, pb.And({item2, item}));
+ state2 = pb.Append(state2, pb.Or({item, item2}));
+ state2 = pb.Append(state2, pb.Or({item2, item}));
+ state2 = pb.Append(state2, pb.Xor({item, item2}));
+ state2 = pb.Append(state2, pb.Xor({item2, item}));
+ return state2;
+ });
+
+ const auto two = pb.Fold(opts, pb.NewEmptyList(optType),
+ [&](TRuntimeNode item2, TRuntimeNode state2) {
+ state2 = pb.Append(state2, pb.And({item, item2}));
+ state2 = pb.Append(state2, pb.Or({item, item2}));
+ state2 = pb.Append(state2, pb.Xor({item, item2}));
+ return state2;
+ });
+
+ return pb.Extend({append, one, two});
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+
+ auto res = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(res.GetListLength(), 66ULL);
+ auto iterator = res.GetListIterator();
+
+ NUdf::TUnboxedValue item;
+
+ /// empty
+ // not
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+
+ /// true
+ // not
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ /// false
+ // not
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+
+ // and
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // or
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ // xor
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFoldWithListInState) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data1 = pb.NewDataLiteral<ui32>(1);
+ auto data2 = pb.NewDataLiteral<ui32>(2);
+ auto data3 = pb.NewDataLiteral<ui32>(3);
+ auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto list = pb.NewList(dataType, {data1, data2, data3});
+ auto optType = pb.NewOptionalType(dataType);
+ auto empty = pb.AddMember(pb.AddMember(
+ pb.NewEmptyStruct(), "Max", pb.NewEmptyOptional(optType)),
+ "List", pb.NewEmptyList(dataType));
+
+ auto pgmReturn = pb.Fold(list, empty,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Max",
+ pb.IfPresent({pb.Member(state, "Max")},
+ [&](TRuntimeNode::TList oldMax) {
+ return pb.NewOptional(pb.Max(oldMax.front(), item));
+ }, pb.NewOptional(item))),
+ "List", pb.Append(pb.Member(state, "List"), item)
+ );
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetElement(0).GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetElement(1).template Get<ui32>(), 3);
+ }
+
+ Y_UNIT_TEST_LLVM(TestManyAppend) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto zeroList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ zeroList = pb.Append(zeroList, pb.NewDataLiteral<ui32>(0));
+ const ui32 n = 13;
+ for (ui32 i = 0; i < n; ++i)
+ zeroList = pb.Extend(zeroList, zeroList);
+
+ auto state = pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter",
+ pb.NewDataLiteral<ui32>(0)), "NewList",
+ pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)));
+
+ auto fold = pb.Fold(zeroList, state,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ Y_UNUSED(item);
+ auto oldList = pb.Member(state, "NewList");
+ auto oldCounter = pb.Member(state, "Counter");
+ return pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter",
+ pb.Add(oldCounter, pb.NewDataLiteral<ui32>(1))),
+ "NewList", pb.Append(oldList, oldCounter));
+ });
+
+ auto pgmReturn = pb.Member(fold, "NewList");
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 1 << n);
+
+ auto iterator = graph->GetValue().GetListIterator();
+ ui32 i = 0;
+ for (NUdf::TUnboxedValue item; iterator.Next(item); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(i, item.template Get<ui32>());
+ }
+ UNIT_ASSERT(!iterator.Skip());
+ UNIT_ASSERT_VALUES_EQUAL(i, 1 << n);
+ }
+
+ Y_UNIT_TEST_LLVM(TestManyPrepend) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto zeroList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ zeroList = pb.Append(zeroList, pb.NewDataLiteral<ui32>(0));
+ const ui32 n = 13;
+ for (ui32 i = 0; i < n; ++i)
+ zeroList = pb.Extend(zeroList, zeroList);
+
+ auto state = pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter",
+ pb.NewDataLiteral<ui32>(0)), "NewList",
+ pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)));
+
+ auto fold = pb.Fold(zeroList, state,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ Y_UNUSED(item);
+ auto oldList = pb.Member(state, "NewList");
+ auto oldCounter = pb.Member(state, "Counter");
+ return pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter",
+ pb.Add(oldCounter, pb.NewDataLiteral<ui32>(1))),
+ "NewList", pb.Prepend(oldCounter, oldList));
+ });
+
+ auto pgmReturn = pb.Member(fold, "NewList");
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 1 << n);
+
+ auto iterator = graph->GetValue().GetListIterator();
+ ui32 i = 1 << n;
+ for (NUdf::TUnboxedValue item; iterator.Next(item);) {
+ UNIT_ASSERT_VALUES_EQUAL(--i, item.template Get<ui32>());
+ }
+ UNIT_ASSERT(!iterator.Skip());
+ UNIT_ASSERT_VALUES_EQUAL(i, 0);
+ }
+
+ Y_UNIT_TEST_LLVM(TestManyExtend) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto zeroList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ zeroList = pb.Append(zeroList, pb.NewDataLiteral<ui32>(0));
+ const ui32 n = 13;
+ for (ui32 i = 0; i < n; ++i)
+ zeroList = pb.Extend(zeroList, zeroList);
+
+ auto state = pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter",
+ pb.NewDataLiteral<ui32>(0)), "NewList",
+ pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id)));
+
+ auto fold = pb.Fold(zeroList, state,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ Y_UNUSED(item);
+ auto oldList = pb.Member(state, "NewList");
+ auto oldCounter = pb.Member(state, "Counter");
+ auto oldCounterMul2 = pb.Mul(oldCounter, pb.NewDataLiteral<ui32>(2));
+ auto extList = pb.NewEmptyList(pb.NewDataType(NUdf::TDataType<ui32>::Id));
+ extList = pb.Append(extList, oldCounterMul2);
+ extList = pb.Append(extList, pb.Increment(oldCounterMul2));
+ return pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Counter",
+ pb.Add(oldCounter, pb.NewDataLiteral<ui32>(1))),
+ "NewList", pb.Extend(oldList, extList));
+ });
+
+ auto pgmReturn = pb.Member(fold, "NewList");
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 1 << (n+1));
+
+ auto iterator = graph->GetValue().GetListIterator();
+ ui32 i = 0;
+ for (NUdf::TUnboxedValue item; iterator.Next(item); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(i, item.template Get<ui32>());
+ }
+ UNIT_ASSERT(!iterator.Skip());
+ UNIT_ASSERT_VALUES_EQUAL(i, 1 << (n + 1));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFoldSingular) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data1 = pb.NewDataLiteral<ui32>(1);
+ auto data2 = pb.NewDataLiteral<ui32>(2);
+ auto data3 = pb.NewDataLiteral<ui32>(3);
+ auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto list = pb.NewList(dataType, {data1, data2, data3});
+ auto fold1 = pb.Fold(list, pb.NewDataLiteral<ui32>(0),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ Y_UNUSED(state);
+ return item;
+ });
+
+ auto fold2 = pb.Fold(list, pb.NewDataLiteral<ui32>(0),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ Y_UNUSED(item);
+ return state;
+ });
+
+ auto pgmReturn = pb.NewList(dataType, {fold1, fold2});
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumListSizes) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto itemType = pb.NewDataType(NUdf::TDataType<float>::Id);
+ auto item = pb.NewDataLiteral<float>(0.f);
+
+ auto listType = pb.NewListType(itemType);
+
+ auto data0 = pb.NewEmptyList(itemType);
+ auto data1 = pb.NewList(itemType, {item});
+ auto data2 = pb.NewList(itemType, {item, item, item});
+ auto data3 = pb.NewList(itemType, {item, item, item, item, item});
+
+ auto list = pb.NewList(listType, {data0, data1, data2, data3});
+
+ auto pgmReturn = pb.Fold1(list,
+ [&](TRuntimeNode item) { return pb.Length(item); },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, pb.Length(item)); }
+ );
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(9ULL, graph->GetValue().template Get<ui64>());
+ }
+
+ Y_UNIT_TEST_LLVM(TestHasListsItems) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto itemType = pb.NewDataType(NUdf::TDataType<float>::Id);
+ auto item = pb.NewDataLiteral<float>(0.f);
+
+ auto listType = pb.NewListType(itemType);
+
+ auto data0 = pb.NewEmptyList(itemType);
+ auto data1 = pb.NewList(itemType, {item});
+ auto data2 = pb.NewEmptyList(itemType);
+
+ auto list = pb.NewList(listType, {data0, data1, data2});
+
+ auto pgmReturn = pb.Fold(list, pb.NewOptional(pb.NewDataLiteral<bool>(false)),
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.Or({state, pb.HasItems(item)}); }
+ );
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT(graph->GetValue().template Get<bool>());
+ }
+
+ Y_UNIT_TEST_LLVM(TestConcat) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("aa");
+ auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("bbb");
+ auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("zzzz");
+ auto type = pb.NewDataType(NUdf::EDataSlot::String);
+ auto list = pb.NewList(type, {data1, data2, data3});
+ auto pgmReturn = pb.Fold(list, data0,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Concat(state, item);
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto res = graph->GetValue();
+ UNBOXED_VALUE_STR_EQUAL(res, "Xaabbbzzzz");
+ }
+
+ Y_UNIT_TEST_LLVM(TestConcatOpt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ auto data0 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>(""));
+ auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("very large string"));
+ auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>(" + "));
+ auto data3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("small"));
+ auto type = pb.NewOptionalType(pb.NewDataType(NUdf::EDataSlot::String));
+ auto list = pb.NewList(type, {data1, data2, data3, data0});
+ auto pgmReturn = pb.Fold(list, data0,
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.Concat(state, item);
+ });
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto res = graph->GetValue();
+ UNBOXED_VALUE_STR_EQUAL(res, "very large string + small");
+ }
+
+ Y_UNIT_TEST_LLVM(TestAggrConcat) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto type = pb.NewOptionalType(pb.NewDataType(NUdf::EDataSlot::Utf8));
+ const auto data0 = pb.NewEmptyOptional(type);
+ const auto data1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("PREFIX:"));
+ const auto data2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very large string"));
+ const auto data3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(":SUFFIX"));
+ const auto list = pb.NewList(type, {data0, data1, data0, data2, data3, data0});
+
+ const auto pgmReturn = pb.Fold1(list,
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrConcat(state, item); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto str = graph->GetValue();
+ UNBOXED_VALUE_STR_EQUAL(str, "PREFIX:very large string:SUFFIX");
+ }
+
+ Y_UNIT_TEST_LLVM(TestLongFold) {
+ for (ui32 i = 0; i < 10; ++i) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const ui32 n = 1000;
+
+ auto firstList = pb.Replicate(pb.NewDataLiteral<ui32>(0),
+ pb.NewDataLiteral<ui64>(n), "", 0, 0);
+
+ auto secondList = pb.Replicate(firstList, pb.NewDataLiteral<ui64>(n), "", 0, 0);
+
+ auto pgmReturn = pb.Fold(secondList, pb.NewDataLiteral<ui32>(0),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ auto partialSum = pb.Fold(item, pb.NewDataLiteral<ui32>(0),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ Y_UNUSED(item);
+ return pb.AggrAdd(state, pb.NewDataLiteral<ui32>(1));
+ });
+
+ return pb.AggrAdd(state, partialSum);
+ });
+
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto value = graph->GetValue().template Get<ui32>();
+ UNIT_ASSERT_VALUES_EQUAL(value, n * n);
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestFoldAggrAddIntervals) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto upper = i64(+1000LL);
+ const auto lower = i64(-1000LL);
+ const auto part = i64(100LL);
+ const auto from = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&lower, sizeof(lower)));
+ const auto stop = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&upper, sizeof(upper)));
+ const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&part, sizeof(part)));
+ const auto list = pb.ListFromRange(from, stop, step);
+
+ const auto pgmReturn = pb.Fold1(pb.ListFromRange(from, stop, step),
+ [&](TRuntimeNode item) { return pb.NewOptional(item); },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(pb.NewOptional(item), state); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto value = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<i64>(), -1000LL);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFoldFoldPerf) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const ui32 n = 3333U;
+
+ const auto firstList = pb.Replicate(pb.NewDataLiteral<ui32>(1), pb.NewDataLiteral<ui64>(n), "", 0, 0);
+
+ const auto secondList = pb.Replicate(firstList, pb.NewDataLiteral<ui64>(n), "", 0, 0);
+
+ const auto pgmReturn = pb.Fold(secondList, pb.NewDataLiteral<ui32>(0),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ const auto partialSum = pb.Fold(item, pb.NewDataLiteral<ui32>(0),
+ [&](TRuntimeNode i2, TRuntimeNode state) {
+ return pb.AggrAdd(state, i2);
+ });
+
+ return pb.AggrAdd(state, partialSum);
+ });
+
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto t2 = TInstant::Now();
+ const auto value = graph->GetValue().template Get<ui32>();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ")." << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value, n * n);
+ }
+
+ std::vector<double> MakeSamples() {
+ std::default_random_engine eng;
+ std::uniform_real_distribution<double> unif(-999.0, +999.0);
+
+ std::vector<double> samples(3333333U);
+
+ eng.seed(std::time(nullptr));
+ std::generate(samples.begin(), samples.end(), std::bind(std::move(unif), std::move(eng)));
+ return samples;
+ }
+
+ static const auto Samples = MakeSamples();
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleArrayListPerf) {
+ TSetup<LLVM> setup;
+
+ const auto t = TInstant::Now();
+ const double sum = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0);
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Fold1(pb.Collect(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); }
+ );
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), sum);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleLazyListPerf) {
+ TSetup<LLVM> setup;
+
+ const auto t = TInstant::Now();
+ const double sum = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0);
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Fold1(pb.LazyList(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); }
+ );
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), sum);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleFilteredArrayListPerf) {
+ TSetup<LLVM> setup;
+
+ const auto t = TInstant::Now();
+ const double sum = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0, [](double s, double v) { return v > 0.0 ? s + v : s; });
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Fold1(
+ pb.Filter(pb.Collect(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); }
+ ),
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); }
+ );
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), sum);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleFilteredLazyListPerf) {
+ TSetup<LLVM> setup;
+
+ const auto t = TInstant::Now();
+ const double sum = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0, [](double s, double v) { return v > 0.0 ? s + v : s; });
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Fold1(
+ pb.Filter(pb.LazyList(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.AggrGreater(item, pb.NewDataLiteral(0.0)); }
+ ),
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); }
+ );
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), sum);
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleArrayListPerf) {
+ TSetup<LLVM> setup;
+
+ double min(Samples.front()), max(Samples.front()), sum(0.0);
+
+ const auto t = TInstant::Now();
+ for (const auto v : Samples) {
+ min = std::fmin(min, v);
+ max = std::fmax(max, v);
+ sum += v;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Fold1(pb.Collect(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.NewTuple({item, item, item}); },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.NewTuple({pb.AggrMin(pb.Nth(state, 0U), item), pb.AggrMax(pb.Nth(state, 1U), item), pb.AggrAdd(pb.Nth(state, 2U), item)});
+ });
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0U).template Get<double>(), min);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1U).template Get<double>(), max);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2U).template Get<double>(), sum);
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleLazyListPerf) {
+ TSetup<LLVM> setup;
+
+ double min(Samples.front()), max(Samples.front()), sum(0.0);
+
+ const auto t = TInstant::Now();
+ for (const auto v : Samples) {
+ min = std::fmin(min, v);
+ max = std::fmax(max, v);
+ sum += v;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Fold1(pb.LazyList(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.NewTuple({item, item, item}); },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.NewTuple({pb.AggrMin(pb.Nth(state, 0U), item), pb.AggrMax(pb.Nth(state, 1U), item), pb.AggrAdd(pb.Nth(state, 2U), item)});
+ });
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0U).template Get<double>(), min);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1U).template Get<double>(), max);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2U).template Get<double>(), sum);
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleFilteredArrayListPerf) {
+ TSetup<LLVM> setup;
+
+ double min(std::nan("")), max(std::nan("")), sum(0.0);
+
+ const auto t = TInstant::Now();
+ for (const auto v : Samples) {
+ if (v < 0.0) {
+ min = std::fmin(min, v);
+ max = std::fmax(max, v);
+ sum += v;
+ }
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Fold1(
+ pb.Filter(pb.Collect(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.AggrLess(item, pb.NewDataLiteral(0.0)); }
+ ),
+ [&](TRuntimeNode item) { return pb.NewTuple({item, item, item}); },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.NewTuple({pb.AggrMin(pb.Nth(state, 0U), item), pb.AggrMax(pb.Nth(state, 1U), item), pb.AggrAdd(pb.Nth(state, 2U), item)});
+ });
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0U).template Get<double>(), min);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1U).template Get<double>(), max);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2U).template Get<double>(), sum);
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleFilteredLazyListPerf) {
+ TSetup<LLVM> setup;
+
+ double min(std::nan("")), max(std::nan("")), sum(0.0);
+
+ const auto t = TInstant::Now();
+ for (const auto v : Samples) {
+ if (v < 0.0) {
+ min = std::fmin(min, v);
+ max = std::fmax(max, v);
+ sum += v;
+ }
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Fold1(
+ pb.Filter(pb.LazyList(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) { return pb.AggrLess(item, pb.NewDataLiteral(0.0)); }
+ ),
+ [&](TRuntimeNode item) { return pb.NewTuple({item, item, item}); },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.NewTuple({pb.AggrMin(pb.Nth(state, 0U), item), pb.AggrMax(pb.Nth(state, 1U), item), pb.AggrAdd(pb.Nth(state, 2U), item)});
+ });
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(0U).template Get<double>(), min);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(1U).template Get<double>(), max);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(2U).template Get<double>(), sum);
+ }
+
+ Y_UNIT_TEST_LLVM(TestAvgDoubleByTupleFoldArrayListPerf) {
+ TSetup<LLVM> setup;
+
+ const auto t = TInstant::Now();
+ const double avg = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0) / Samples.size();
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto fold = pb.Fold(pb.Collect(TRuntimeNode(list, false)),
+ pb.NewTuple({pb.NewDataLiteral(0.0), pb.NewDataLiteral<ui64>(0ULL)}),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0), item), pb.Increment(pb.Nth(state, 1))});
+ });
+
+ const auto pgmReturn = pb.Div(pb.Nth(fold, 0U), pb.Nth(fold, 1U));
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), avg);
+ }
+
+ Y_UNIT_TEST_LLVM(TestAvgDoubleByTupleFoldLazyListPerf) {
+ TSetup<LLVM> setup;
+
+ const auto t = TInstant::Now();
+ const double avg = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0) / Samples.size();
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto fold = pb.Fold(pb.LazyList(TRuntimeNode(list, false)),
+ pb.NewTuple({pb.NewDataLiteral(0.0), pb.NewDataLiteral<ui64>(0ULL)}),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.NewTuple({pb.AggrAdd(pb.Nth(state, 0U), item), pb.Increment(pb.Nth(state, 1U))});
+ });
+
+ const auto pgmReturn = pb.Div(pb.Nth(fold, 0U), pb.Nth(fold, 1U));
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), avg);
+ }
+
+ Y_UNIT_TEST_LLVM(TestAvgDoubleByCollectFoldLazyListPerf) {
+ TSetup<LLVM> setup;
+
+ const auto t = TInstant::Now();
+ const double avg = std::accumulate(Samples.cbegin(), Samples.cend(), 0.0) / Samples.size();
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto src = pb.Collect(pb.LazyList(TRuntimeNode(list, false)));
+ const auto pgmReturn = pb.Div(
+ pb.Fold(src, pb.NewDataLiteral(0.0),
+ [&](TRuntimeNode item, TRuntimeNode state) { return pb.AggrAdd(state, item); }
+ ),
+ pb.Length(src)
+ );
+
+ const auto t1 = TInstant::Now();
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(Samples.size(), items));
+ std::transform(Samples.cbegin(), Samples.cend(), items, &ToValue<double>);
+ const auto t2 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t3 = TInstant::Now();
+ Cerr << "Time is " << t3 - t1 << " (" << t2 - t1 << " + " << t3 - t2 << ") vs C++ " << cppTime << Endl;
+ UNIT_ASSERT_VALUES_EQUAL(value.template Get<double>(), avg);
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp
new file mode 100644
index 00000000000..1acb4460911
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_fromstring_ut.cpp
@@ -0,0 +1,38 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLFromStringTest) {
+ Y_UNIT_TEST_LLVM(TestFromString) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto data = pb.NewDataLiteral<NUdf::EDataSlot::String>("abcdefg");
+ const auto pgmReturn1 = pb.StrictFromString(data, pb.NewDecimalType(10, 7));
+ const auto graph1 = setup.BuildGraph(pgmReturn1);
+ UNIT_ASSERT_EXCEPTION_CONTAINS(graph1->GetValue(), std::exception, R"(Terminate was called, reason(45): could not convert "abcdefg" to Decimal(10, 7))");
+
+ const auto pgmReturn2 = pb.StrictFromString(data, pb.NewDataType(NUdf::TDataType<ui64>::Id));
+ const auto graph2 = setup.BuildGraph(pgmReturn2);
+ UNIT_ASSERT_EXCEPTION_CONTAINS(graph2->GetValue(), std::exception, R"(Terminate was called, reason(37): could not convert "abcdefg" to Uint64)");
+ }
+
+ Y_UNIT_TEST_LLVM(TestFromStringHugePayload) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ const auto data = pb.NewDataLiteral<NUdf::EDataSlot::String>(TString(10000, 'x'));
+ const auto pgmReturn1 = pb.StrictFromString(data, pb.NewDecimalType(10, 7));
+ const auto graph1 = setup.BuildGraph(pgmReturn1);
+ UNIT_ASSERT_EXCEPTION_CONTAINS(graph1->GetValue(), std::exception, R"(Terminate was called, reason(5050): could not convert "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" (truncated) to Decimal(10, 7))");
+
+ const auto pgmReturn2 = pb.StrictFromString(data, pb.NewDataType(NUdf::TDataType<ui64>::Id));
+ const auto graph2 = setup.BuildGraph(pgmReturn2);
+ UNIT_ASSERT_EXCEPTION_CONTAINS(graph2->GetValue(), std::exception, R"(Terminate was called, reason(5042): could not convert "xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx" (truncated) to Uint64)");
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp
new file mode 100644
index 00000000000..6f655ef78b1
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_grace_join_ut.cpp
@@ -0,0 +1,2003 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+#include <ydb/library/yql/minikql/comp_nodes/mkql_grace_join_imp.h>
+
+
+#include <chrono>
+#include <iostream>
+#include <cstring>
+#include <vector>
+#include <cassert>
+#include <cstdlib>
+#include <stdlib.h>
+
+#include <util/system/compiler.h>
+#include <util/stream/null.h>
+#include <util/system/mem_info.h>
+
+#include <cstdint>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+constexpr bool IsVerbose = false;
+#define CTEST (IsVerbose ? Cerr : Cnull)
+
+
+Y_UNIT_TEST_SUITE(TMiniKQLGraceJoinMemTest) {
+ Y_UNIT_TEST(TestMem1) {
+
+ const ui64 TupleSize = 1024;
+ const ui64 NBuckets = 128;
+ const ui64 NTuples = 100000;
+ const ui64 BucketSize = (2* NTuples * (TupleSize + 1) ) / NBuckets;
+
+ ui64 *bigTuple = (ui64 * ) malloc(TupleSize * sizeof(ui64));
+ ui64 *buckets[NBuckets];
+ ui64 tuplesPos[NBuckets];
+
+ for (ui64 i = 0; i < TupleSize; i++)
+ {
+ bigTuple[i] = std::rand() / (RAND_MAX / 10000);
+ }
+
+ ui64 bucket = 0;
+ ui64 milliseconds = 0;
+
+ const ui64 BitsForData = 30;
+
+ char* a = (char * )malloc(1 << BitsForData);
+ char* b = (char *) malloc(1 << BitsForData);
+ UNIT_ASSERT(a);
+ UNIT_ASSERT(b);
+
+ memset(a, 1, 1 << BitsForData);
+ memset(b, 2, 1 << BitsForData);
+
+ std::chrono::steady_clock::time_point begin01 = std::chrono::steady_clock::now();
+
+ memcpy(b, a, 1 << BitsForData);
+
+ std::chrono::steady_clock::time_point end01 = std::chrono::steady_clock::now();
+
+ UNIT_ASSERT(*a == 1);
+ UNIT_ASSERT(*b == 1);
+
+ Y_DO_NOT_OPTIMIZE_AWAY(a);
+ Y_DO_NOT_OPTIMIZE_AWAY(b);
+
+ ui64 microseconds = std::chrono::duration_cast<std::chrono::microseconds>(end01 - begin01).count();
+ CTEST << "Time for memcpy = " << microseconds << "[microseconds]" << Endl;
+ CTEST << "Data size = " << (1<<BitsForData) / (1024 * 1024) << "[MB]" << Endl;
+ CTEST << "Memcpy speed = " << ( (1<<BitsForData) ) / (microseconds) << "MB/sec" << Endl;
+ CTEST << Endl;
+
+ std::vector<std::vector<ui64>> vec_buckets;
+ vec_buckets.resize(NBuckets);
+ for (ui64 i = 0; i < NBuckets; i++)
+ {
+ vec_buckets[i].resize(2 * TupleSize * NTuples / (NBuckets - 1), 0);
+ vec_buckets[i].clear();
+// vec_buckets[i].reserve( 2 * TupleSize * NTuples / (NBuckets - 1));
+ }
+
+ for (ui64 i = 0; i < NBuckets; i++) {
+ buckets[i] = (ui64 * ) malloc( (BucketSize * sizeof(ui64) * 32) / 32);
+ memset( buckets[i], 1, (BucketSize * sizeof(ui64) * 32) / 32);
+ tuplesPos[i] = 0;
+ }
+
+
+ std::chrono::steady_clock::time_point begin02 = std::chrono::steady_clock::now();
+
+ for (ui64 i = 0; i < NTuples; i++)
+ {
+ bucket = i % NBuckets;
+// bucket = std::rand() / ( RAND_MAX / (NBuckets-1));
+ std::vector<ui64> &curr_vec = vec_buckets[bucket];
+ curr_vec.insert(curr_vec.end(), bigTuple, bigTuple + TupleSize);
+ }
+
+ std::chrono::steady_clock::time_point end02 = std::chrono::steady_clock::now();
+
+ milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end02 - begin02).count();
+ CTEST << "Time for std::insert = " << milliseconds << "[ms]" << Endl;
+ CTEST << "Total MB = " << (TupleSize * NTuples * sizeof(ui64) / (1024 * 1024)) << Endl;
+ CTEST << "std::insert speed = " << (TupleSize * NTuples * sizeof(ui64) * 1000) / (milliseconds * 1024 * 1024) << "MB/sec" << Endl;
+ CTEST << Endl;
+
+ std::chrono::steady_clock::time_point begin03 = std::chrono::steady_clock::now();
+
+ for (ui64 i = 0; i < NTuples; i++)
+ {
+
+ bucket = i % NBuckets;
+// bucket = std::rand() / ( RAND_MAX / (NBuckets-1));
+
+ ui64 * dst = buckets[bucket] + tuplesPos[bucket];
+ std::memcpy(dst, bigTuple, TupleSize*sizeof(ui64));
+ tuplesPos[bucket] += TupleSize;
+ }
+
+ std::chrono::steady_clock::time_point end03 = std::chrono::steady_clock::now();
+
+ milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end03 - begin03).count();
+ CTEST << "Time for std::memcpy = " << milliseconds << "[ms]" << Endl;
+ CTEST << "Total MB = " << (TupleSize * NTuples * sizeof(ui64) / (1024 * 1024)) << Endl;
+ CTEST << "std:memcpy speed = " << (TupleSize * NTuples * sizeof(ui64) * 1000) / (milliseconds * 1024 * 1024) << "MB/sec" << Endl;
+ CTEST << Endl;
+
+ for (ui64 i = 0; i < NBuckets; i++) {
+ tuplesPos[i] = 0;
+ }
+
+
+ std::chrono::steady_clock::time_point begin04 = std::chrono::steady_clock::now();
+
+ for (ui64 i = 0; i < NTuples; i++)
+ {
+ bucket = std::rand() / ( RAND_MAX / (NBuckets-1));
+
+ ui64 * dst = buckets[bucket] + tuplesPos[bucket];
+
+ ui64 *dst1 = dst + 1;
+ ui64 *dst2 = dst + 2;
+ ui64 *dst3 = dst + 3;
+ ui64 *src = bigTuple;
+ ui64 *src1 = bigTuple + 1;
+ ui64 *src2 = bigTuple + 2;
+ ui64 *src3 = bigTuple + 3;
+
+ for (ui64 i = 0; i < TupleSize; i += 4)
+ {
+ *dst++ = *src++;
+ *dst1++ = *src1++;
+ *dst2++ = *src2++;
+ *dst3++ = *src3++;
+
+ }
+ tuplesPos[bucket] += TupleSize;
+ }
+
+ std::chrono::steady_clock::time_point end04 = std::chrono::steady_clock::now();
+
+ milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end04 - begin04).count();
+ CTEST << "Time for loop copy = " << milliseconds << "[ms]" << Endl;
+ CTEST << "Total MB = " << (TupleSize * NTuples * sizeof(ui64) / (1024 * 1024)) << Endl;
+ CTEST << "Loop copy speed = " << (TupleSize * NTuples * sizeof(ui64) * 1000) / (milliseconds * 1024 * 1024) << "MB/sec" << Endl;
+ CTEST << Endl;
+
+ for (ui64 i = 0; i < NBuckets; i++) {
+ free(buckets[i]);
+ }
+
+ free(b);
+ free(a);
+ free(bigTuple);
+
+
+ UNIT_ASSERT(true);
+
+ }
+
+}
+
+
+Y_UNIT_TEST_SUITE(TMiniKQLGraceJoinImpTest) {
+ Y_UNIT_TEST_LLVM(TestImp1) {
+ TSetup<LLVM> setup;
+ ui64 tuple[11] = {0,1,2,3,4,5,6,7,8,9,10};
+ ui32 strSizes[2] = {4, 4};
+ char * strVals[] = {(char *)"aaaaa", (char *)"bbbb"};
+
+ char * bigStrVal[] = {(char *)"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ (char *)"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"};
+ ui32 bigStrSize[2] = {151, 151};
+
+
+ NMemInfo::TMemInfo mi = NMemInfo::GetMemInfo();
+ CTEST << "Mem usage before tables tuples added (MB): " << mi.RSS / (1024 * 1024) << Endl;
+
+ GraceJoin::TTable bigTable(1,1,1,1);
+ GraceJoin::TTable smallTable(1,1,1,1);
+ GraceJoin::TTable joinTable(1,1,1,1);
+
+ std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
+
+ const ui64 TupleSize = 1024;
+
+ ui64 bigTuple[TupleSize];
+
+ for (ui64 i = 0; i < TupleSize; i++) {
+ bigTuple[i] = std::rand() / ( RAND_MAX / 10000 );
+ }
+
+ ui64 milliseconds = 0;
+
+
+
+ const ui64 BigTableTuples = 600000;
+ const ui64 SmallTableTuples = 150000;
+ const ui64 BigTupleSize = 40;
+
+ std::chrono::steady_clock::time_point begin03 = std::chrono::steady_clock::now();
+
+
+ for ( ui64 i = 0; i < BigTableTuples; i++) {
+ tuple[1] = std::rand() % SmallTableTuples;
+ tuple[2] = tuple[1];
+ bigTable.AddTuple(tuple, strVals, strSizes);
+ }
+
+ smallTable.AddTuple(tuple, bigStrVal, bigStrSize);
+
+ for ( ui64 i = 0; i < SmallTableTuples + 1; i++) {
+ tuple[1] = std::rand() % SmallTableTuples;
+ tuple[2] = tuple[1];
+ smallTable.AddTuple(tuple, strVals, strSizes);
+ }
+
+ std::chrono::steady_clock::time_point end03 = std::chrono::steady_clock::now();
+ milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end03 - begin03).count();
+ CTEST << "Time for hash = " << milliseconds << "[ms]" << Endl;
+ CTEST << "Adding tuples speed: " << (BigTupleSize * (BigTableTuples + SmallTableTuples) * 1000) / ( milliseconds * 1024 * 1024) << "MB/sec" << Endl;
+ CTEST << Endl;
+
+ mi = NMemInfo::GetMemInfo();
+ CTEST << "Mem usage after tables tuples added (MB): " << mi.RSS / (1024 * 1024) << Endl;
+
+
+ bigTable.Clear();
+ smallTable.Clear();
+
+ begin03 = std::chrono::steady_clock::now();
+
+
+ for ( ui64 i = 0; i < BigTableTuples; i++) {
+ tuple[1] = std::rand() % SmallTableTuples;
+ tuple[2] = tuple[1];
+ bigTable.AddTuple(tuple, strVals, strSizes);
+ }
+
+ smallTable.AddTuple(tuple, bigStrVal, bigStrSize);
+
+ for ( ui64 i = 0; i < SmallTableTuples + 1; i++) {
+ tuple[1] = std::rand() % SmallTableTuples;
+ tuple[2] = tuple[1];
+ smallTable.AddTuple(tuple, strVals, strSizes);
+ }
+
+ end03 = std::chrono::steady_clock::now();
+ milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end03 - begin03).count();
+ CTEST << "Time for hash = " << milliseconds << "[ms]" << Endl;
+ CTEST << "Adding tuples speed: " << (BigTupleSize * (BigTableTuples + SmallTableTuples) * 1000) / ( milliseconds * 1024 * 1024) << "MB/sec" << Endl;
+ CTEST << Endl;
+
+ mi = NMemInfo::GetMemInfo();
+ CTEST << "Mem usage after tables tuples added (MB): " << mi.RSS / (1024 * 1024) << Endl;
+
+
+ std::vector<ui64> vals1, vals2;
+ std::vector<char *> strVals1, strVals2;
+ std::vector<ui32> strSizes1, strSizes2;
+ GraceJoin::TupleData td1, td2;
+ vals1.resize(100);
+ vals2.resize(100);
+ strVals1.resize(100);
+ strVals2.resize(100);
+ strSizes1.resize(100);
+ strSizes2.resize(100);
+ td1.IntColumns = vals1.data();
+ td1.StrColumns = strVals1.data();
+ td1.StrSizes = strSizes1.data();
+ td2.IntColumns = vals2.data();
+ td2.StrColumns = strVals2.data();
+ td2.StrSizes = strSizes2.data();
+
+ ui64 numBigTuples = 0;
+ bigTable.ResetIterator();
+
+ std::chrono::steady_clock::time_point begin04 = std::chrono::steady_clock::now();
+
+ while(bigTable.NextTuple(td1)) { numBigTuples++; }
+
+ CTEST << "Num of big tuples 1: " << numBigTuples << Endl;
+
+ std::chrono::steady_clock::time_point end04 = std::chrono::steady_clock::now();
+ CTEST << "Time for get 1 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end04 - begin04).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+ numBigTuples = 0;
+ bigTable.ResetIterator();
+
+ std::chrono::steady_clock::time_point begin041 = std::chrono::steady_clock::now();
+
+ while(bigTable.NextTuple(td2)) { numBigTuples++; }
+
+ CTEST << "Num of big tuples 2: " << numBigTuples << Endl;
+
+ std::chrono::steady_clock::time_point end041 = std::chrono::steady_clock::now();
+ CTEST << "Time for get 2 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end041 - begin041).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+
+ std::chrono::steady_clock::time_point begin05 = std::chrono::steady_clock::now();
+
+ joinTable.Join(smallTable,bigTable);
+
+ std::chrono::steady_clock::time_point end05 = std::chrono::steady_clock::now();
+ CTEST << "Time for join = " << std::chrono::duration_cast<std::chrono::milliseconds>(end05 - begin05).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+ mi = NMemInfo::GetMemInfo();
+ CTEST << "Mem usage after tables join (MB): " << mi.RSS / (1024 * 1024) << Endl;
+
+
+ joinTable.ResetIterator();
+ ui64 numJoinedTuples = 0;
+
+
+ std::chrono::steady_clock::time_point begin042 = std::chrono::steady_clock::now();
+
+ while(joinTable.NextJoinedData(td1, td2)) { numJoinedTuples++; }
+
+ CTEST << "Num of joined tuples : " << numJoinedTuples << Endl;
+
+ std::chrono::steady_clock::time_point end042 = std::chrono::steady_clock::now();
+ CTEST << "Time for get joined tuples: = " << std::chrono::duration_cast<std::chrono::milliseconds>(end042 - begin042).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+
+ std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
+ CTEST << "Time difference = " << std::chrono::duration_cast<std::chrono::milliseconds>(end - begin).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLGraceJoinAnyTest) {
+ Y_UNIT_TEST_LLVM(TestImp2) {
+ TSetup<LLVM> setup;
+ ui64 tuple[11] = {0,1,2,3,4,5,6,7,8,9,10};
+ ui32 strSizes[2] = {4, 4};
+ char * strVals[] = {(char *)"aaaaa", (char *)"bbbb"};
+
+ char * bigStrVal[] = {(char *)"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ (char *)"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"};
+ ui32 bigStrSize[2] = {151, 151};
+
+
+
+ GraceJoin::TTable bigTable (1,1,1,1,0,0,1, nullptr, true);
+ GraceJoin::TTable smallTable(1,1,1,1,0,0,1, nullptr, true);
+ GraceJoin::TTable joinTable (1,1,1,1,0,0,1, nullptr, true);
+
+ std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
+
+ const ui64 TupleSize = 1024;
+
+ ui64 bigTuple[TupleSize];
+
+ for (ui64 i = 0; i < TupleSize; i++) {
+ bigTuple[i] = std::rand() / ( RAND_MAX / 10000 );
+ }
+
+ ui64 milliseconds = 0;
+
+
+
+ const ui64 BigTableTuples = 600000;
+ const ui64 SmallTableTuples = 150000;
+ const ui64 BigTupleSize = 40;
+
+ std::chrono::steady_clock::time_point begin03 = std::chrono::steady_clock::now();
+
+
+ for ( ui64 i = 0; i < BigTableTuples; i++) {
+ tuple[1] = i % SmallTableTuples;
+ tuple[2] = tuple[1];
+ bigTable.AddTuple(tuple, strVals, strSizes);
+ }
+
+ smallTable.AddTuple(tuple, bigStrVal, bigStrSize);
+
+ for ( ui64 i = 0; i < SmallTableTuples + 1; i++) {
+ tuple[1] = i;
+ tuple[2] = tuple[1];
+ smallTable.AddTuple(tuple, strVals, strSizes);
+ }
+
+ std::chrono::steady_clock::time_point end03 = std::chrono::steady_clock::now();
+ milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end03 - begin03).count();
+ CTEST << "Time for hash = " << milliseconds << "[ms]" << Endl;
+ CTEST << "Adding tuples speed: " << (BigTupleSize * (BigTableTuples + SmallTableTuples) * 1000) / ( milliseconds * 1024 * 1024) << "MB/sec" << Endl;
+ CTEST << Endl;
+
+ std::vector<ui64> vals1, vals2;
+ std::vector<char *> strVals1, strVals2;
+ std::vector<ui32> strSizes1, strSizes2;
+ GraceJoin::TupleData td1, td2;
+ vals1.resize(100);
+ vals2.resize(100);
+ strVals1.resize(100);
+ strVals2.resize(100);
+ strSizes1.resize(100);
+ strSizes2.resize(100);
+ td1.IntColumns = vals1.data();
+ td1.StrColumns = strVals1.data();
+ td1.StrSizes = strSizes1.data();
+ td2.IntColumns = vals2.data();
+ td2.StrColumns = strVals2.data();
+ td2.StrSizes = strSizes2.data();
+
+ ui64 numBigTuples = 0;
+ bigTable.ResetIterator();
+
+ std::chrono::steady_clock::time_point begin04 = std::chrono::steady_clock::now();
+
+ while(bigTable.NextTuple(td1)) { numBigTuples++; }
+
+ CTEST << "Num of big tuples 1: " << numBigTuples << Endl;
+
+ std::chrono::steady_clock::time_point end04 = std::chrono::steady_clock::now();
+ CTEST << "Time for get 1 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end04 - begin04).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+ numBigTuples = 0;
+ bigTable.ResetIterator();
+
+ std::chrono::steady_clock::time_point begin041 = std::chrono::steady_clock::now();
+
+ while(bigTable.NextTuple(td2)) { numBigTuples++; }
+
+ CTEST << "Num of big tuples 2: " << numBigTuples << Endl;
+
+ std::chrono::steady_clock::time_point end041 = std::chrono::steady_clock::now();
+ CTEST << "Time for get 2 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end041 - begin041).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+
+ std::chrono::steady_clock::time_point begin05 = std::chrono::steady_clock::now();
+
+ joinTable.Join(smallTable,bigTable);
+
+ std::chrono::steady_clock::time_point end05 = std::chrono::steady_clock::now();
+ CTEST << "Time for join = " << std::chrono::duration_cast<std::chrono::milliseconds>(end05 - begin05).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+ joinTable.ResetIterator();
+ ui64 numJoinedTuples = 0;
+
+
+ std::chrono::steady_clock::time_point begin042 = std::chrono::steady_clock::now();
+
+ while(joinTable.NextJoinedData(td1, td2)) { numJoinedTuples++; }
+
+ CTEST << "Num of joined tuples : " << numJoinedTuples << Endl;
+
+ std::chrono::steady_clock::time_point end042 = std::chrono::steady_clock::now();
+ CTEST << "Time for get joined tuples: = " << std::chrono::duration_cast<std::chrono::milliseconds>(end042 - begin042).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+
+ std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
+ CTEST << "Time difference = " << std::chrono::duration_cast<std::chrono::milliseconds>(end - begin).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLGraceSelfJoinTest) {
+ Y_UNIT_TEST_LLVM(TestImp3) {
+ TSetup<LLVM> setup;
+ ui64 tuple[11] = {0,1,2,3,4,5,6,7,8,9,10};
+ ui32 strSizes[2] = {4, 4};
+ char * strVals[] = {(char *)"aaaaa", (char *)"bbbb"};
+
+ char * bigStrVal[] = {(char *)"aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+ (char *)"bbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbbb"};
+ ui32 bigStrSize[2] = {151, 151};
+
+
+
+ GraceJoin::TTable bigTable (1,1,1,1,0,0,1, nullptr, false);
+ GraceJoin::TTable smallTable(1,1,1,1,0,0,1, nullptr, false);
+ GraceJoin::TTable joinTable (1,1,1,1,0,0,1, nullptr, false);
+
+ std::chrono::steady_clock::time_point begin = std::chrono::steady_clock::now();
+
+ const ui64 TupleSize = 1024;
+
+ ui64 bigTuple[TupleSize];
+
+ for (ui64 i = 0; i < TupleSize; i++) {
+ bigTuple[i] = std::rand() / ( RAND_MAX / 10000 );
+ }
+
+ ui64 milliseconds = 0;
+
+
+
+ const ui64 BigTableTuples = 600000;
+ const ui64 SmallTableTuples = 150000;
+ const ui64 BigTupleSize = 40;
+
+ std::chrono::steady_clock::time_point begin03 = std::chrono::steady_clock::now();
+
+
+ for ( ui64 i = 0; i < BigTableTuples; i++) {
+ tuple[1] = i % SmallTableTuples;
+ tuple[2] = tuple[1];
+ bigTable.AddTuple(tuple, strVals, strSizes);
+ }
+
+ smallTable.AddTuple(tuple, bigStrVal, bigStrSize);
+
+ for ( ui64 i = 0; i < SmallTableTuples + 1; i++) {
+ tuple[1] = i;
+ tuple[2] = tuple[1];
+ smallTable.AddTuple(tuple, strVals, strSizes);
+ }
+
+ std::chrono::steady_clock::time_point end03 = std::chrono::steady_clock::now();
+ milliseconds = std::chrono::duration_cast<std::chrono::milliseconds>(end03 - begin03).count();
+ CTEST << "Time for hash = " << milliseconds << "[ms]" << Endl;
+ CTEST << "Adding tuples speed: " << (BigTupleSize * (BigTableTuples + SmallTableTuples) * 1000) / ( milliseconds * 1024 * 1024) << "MB/sec" << Endl;
+ CTEST << Endl;
+
+ std::vector<ui64> vals1, vals2;
+ std::vector<char *> strVals1, strVals2;
+ std::vector<ui32> strSizes1, strSizes2;
+ GraceJoin::TupleData td1, td2;
+ vals1.resize(100);
+ vals2.resize(100);
+ strVals1.resize(100);
+ strVals2.resize(100);
+ strSizes1.resize(100);
+ strSizes2.resize(100);
+ td1.IntColumns = vals1.data();
+ td1.StrColumns = strVals1.data();
+ td1.StrSizes = strSizes1.data();
+ td2.IntColumns = vals2.data();
+ td2.StrColumns = strVals2.data();
+ td2.StrSizes = strSizes2.data();
+
+ ui64 numBigTuples = 0;
+ bigTable.ResetIterator();
+
+ std::chrono::steady_clock::time_point begin04 = std::chrono::steady_clock::now();
+
+ while(bigTable.NextTuple(td1)) { numBigTuples++; }
+
+ CTEST << "Num of big tuples 1: " << numBigTuples << Endl;
+
+ std::chrono::steady_clock::time_point end04 = std::chrono::steady_clock::now();
+ CTEST << "Time for get 1 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end04 - begin04).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+ numBigTuples = 0;
+ bigTable.ResetIterator();
+
+ std::chrono::steady_clock::time_point begin041 = std::chrono::steady_clock::now();
+
+ while(bigTable.NextTuple(td2)) { numBigTuples++; }
+
+ CTEST << "Num of big tuples 2: " << numBigTuples << Endl;
+
+ std::chrono::steady_clock::time_point end041 = std::chrono::steady_clock::now();
+ CTEST << "Time for get 2 = " << std::chrono::duration_cast<std::chrono::milliseconds>(end041 - begin041).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+
+ std::chrono::steady_clock::time_point begin05 = std::chrono::steady_clock::now();
+
+ joinTable.Join(bigTable,bigTable);
+
+ std::chrono::steady_clock::time_point end05 = std::chrono::steady_clock::now();
+ CTEST << "Time for join = " << std::chrono::duration_cast<std::chrono::milliseconds>(end05 - begin05).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+ joinTable.ResetIterator();
+ ui64 numJoinedTuples = 0;
+
+
+ std::chrono::steady_clock::time_point begin042 = std::chrono::steady_clock::now();
+
+ while(joinTable.NextJoinedData(td1, td2)) { numJoinedTuples++; }
+
+ CTEST << "Num of joined tuples : " << numJoinedTuples << Endl;
+
+ std::chrono::steady_clock::time_point end042 = std::chrono::steady_clock::now();
+ CTEST << "Time for get joined tuples: = " << std::chrono::duration_cast<std::chrono::milliseconds>(end042 - begin042).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+
+ std::chrono::steady_clock::time_point end = std::chrono::steady_clock::now();
+ CTEST << "Time difference = " << std::chrono::duration_cast<std::chrono::milliseconds>(end - begin).count() << "[ms]" << Endl;
+ CTEST << Endl;
+
+
+ }
+}
+
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 40u
+Y_UNIT_TEST_SUITE(TMiniKQLSelfJoinTest) {
+
+ Y_UNIT_TEST_LLVM(TestInner1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(4);
+ const auto key4 = pb.NewDataLiteral<ui32>(4);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3}),
+ pb.NewTuple({key4, payload4})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.SelfJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::Inner, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "C");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "C");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "B");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "A");
+ UNIT_ASSERT(!iterator.Next(tuple));
+
+ }
+
+
+ }
+
+ Y_UNIT_TEST_LLVM(TestDiffKeys) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(4);
+ const auto key4 = pb.NewDataLiteral<ui32>(4);
+ const auto key11 = pb.NewDataLiteral<ui32>(1);
+ const auto key21 = pb.NewDataLiteral<ui32>(1);
+ const auto key31 = pb.NewDataLiteral<ui32>(2);
+ const auto key41 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, key11, payload1}),
+ pb.NewTuple({key2, key21, payload2}),
+ pb.NewTuple({key3, key31, payload3}),
+ pb.NewTuple({key4, key41, payload4})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.SelfJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ EJoinKind::Inner, {0U}, {1U}, {2U, 0U}, {2U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "C");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "A");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "B");
+ UNIT_ASSERT(!iterator.Next(tuple));
+
+ }
+
+
+ }
+
+
+}
+#endif
+
+Y_UNIT_TEST_SUITE(TMiniKQLGraceJoinTest) {
+
+ Y_UNIT_TEST_LLVM(TestInner1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(4);
+ const auto key4 = pb.NewDataLiteral<ui32>(4);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::Inner, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(!iterator.Next(tuple));
+
+ }
+
+
+ }
+
+ Y_UNIT_TEST_LLVM(TestInnerDoubleCondition1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(4);
+ const auto key4 = pb.NewDataLiteral<ui32>(4);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType1 = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto tupleType2 = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+
+ const auto list1 = pb.NewList(tupleType1, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType2, {
+ pb.NewTuple({key2, key2, payload4}),
+ pb.NewTuple({key3, key2, payload5}),
+ pb.NewTuple({key4, key1, payload6})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ EJoinKind::Inner, {0U, 0U}, {0U, 1U}, {1U, 0U}, {2U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(!iterator.Next(tuple));
+
+ }
+
+
+ }
+
+ Y_UNIT_TEST_LLVM(TestInnerStringKey1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("1");
+ const auto key2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("2");
+ const auto key3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("4");
+ const auto key4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("4");
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::Inner, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(!iterator.Next(tuple));
+
+ }
+
+
+ }
+
+
+
+ Y_UNIT_TEST_LLVM(TMiniKQLGraceJoinTestInnerMulti1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::Inner, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeft1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(3);
+ const auto key4 = pb.NewDataLiteral<ui32>(4);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::Left, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+
+ for (ui32 i = 0; i < 3; i++) {
+ iterator.Next(tuple);
+ const auto cell = (tuple.GetElement(0));
+ if (cell.AsStringRef() == "A") {
+ UNIT_ASSERT(!tuple.GetElement(1));
+ }
+ if (cell.AsStringRef() == "B") {
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ }
+ if (cell.AsStringRef() == "C") {
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ }
+ }
+
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftMulti1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::Left, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftSemi1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::LeftSemi, {0U}, {0U}, {1U, 0U, 0U, 1U}, {}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftOnly1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto key5 = pb.NewDataLiteral<ui32>(4);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("D");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3}),
+ pb.NewTuple({key4, payload4}),
+ pb.NewTuple({key5, payload4})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload5}),
+ pb.NewTuple({key3, payload6}),
+ pb.NewTuple({key4, payload7})
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::LeftOnly, {0U}, {0U}, {1U, 0U, 0U, 1U}, {}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "D");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 4);
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 1);
+
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftSemiWithNullKey1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3));
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id, true),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload4}),
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload3}),
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::LeftSemi, {0U}, {0U}, {1U, 0U, 0U, 1U}, {}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftOnlyWithNullKey1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3));
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id, true),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload4}),
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload3}),
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::LeftOnly, {0U}, {0U}, {1U, 0U, 0U, 1U}, {}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 1);
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestRight1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(3);
+ const auto key4 = pb.NewDataLiteral<ui32>(4);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::Right, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+
+ for (ui32 i = 0; i < 3; i++) {
+ iterator.Next(tuple);
+ const auto cell = (tuple.GetElement(1));
+ if (cell.AsStringRef() == "Z") {
+ UNIT_ASSERT(!tuple.GetElement(0));
+ }
+ if (cell.AsStringRef() == "X") {
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ }
+ if (cell.AsStringRef() == "Y") {
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ }
+ }
+
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+
+ }
+ }
+
+
+ Y_UNIT_TEST_LLVM(TestRightOnly1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::RightOnly, {0U}, {0U}, {}, {1U, 0U, 0U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "Z");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 3);
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+
+
+ Y_UNIT_TEST_LLVM(TestRightSemi1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::RightSemi, {0U}, {0U}, {}, {1U, 0U, 0U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "Y");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+
+ Y_UNIT_TEST_LLVM(TestRightMulti1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::Right, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(!tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+
+ Y_UNIT_TEST_LLVM(TestRightSemiWithNullKey1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3));
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id, true),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload4}),
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload3}),
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::RightSemi, {0U}, {0U}, {}, {1U, 0U, 0U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "Y");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestRightOnlyWithNullKey1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3));
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id, true),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload4}),
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload3}),
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::RightOnly, {0U}, {0U}, {}, {1U, 0U, 0U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "Z");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestFull1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::Full, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z");
+ UNIT_ASSERT(!tuple.GetElement(0));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+
+ Y_UNIT_TEST_LLVM(TestExclusion1) {
+
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.GraceJoin(
+ pb.ExpandMap(pb.ToFlow(list1), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.ExpandMap(pb.ToFlow(list2), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ EJoinKind::Exclusion, {0U}, {0U}, {1U, 0U}, {1U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z");
+ UNIT_ASSERT(!tuple.GetElement(0));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+}
+
+
+}
+
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp
new file mode 100644
index 00000000000..1833bbf5095
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp
@@ -0,0 +1,254 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+template<bool UseLLVM>
+TRuntimeNode MakeStream(TSetup<UseLLVM>& setup, ui64 count = 9U) {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ TCallableBuilder callableBuilder(*setup.Env, "TestStream",
+ pgmBuilder.NewStreamType(
+ pgmBuilder.NewDataType(NUdf::EDataSlot::Uint64)
+ )
+ );
+
+ callableBuilder.Add(pgmBuilder.NewDataLiteral(count));
+
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
+template<bool UseLLVM>
+TRuntimeNode Group(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch,
+ const std::function<TRuntimeNode(TRuntimeNode)>& handler = {})
+{
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto keyExtractor = [&](TRuntimeNode item) {
+ return item;
+ };
+
+ stream = pgmBuilder.GroupingCore(stream, groupSwitch, keyExtractor, handler);
+ return pgmBuilder.FlatMap(stream, [&](TRuntimeNode grpItem) {
+ return pgmBuilder.Squeeze(pgmBuilder.Nth(grpItem, 1),
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("*"),
+ [&] (TRuntimeNode item, TRuntimeNode state) {
+ auto res = pgmBuilder.Concat(pgmBuilder.ToString(pgmBuilder.Nth(grpItem, 0)), pgmBuilder.ToString(item));
+ res = pgmBuilder.Concat(state, res);
+ res = pgmBuilder.Concat(res, pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("*"));
+ return res;
+ },
+ {}, {});
+ });
+}
+
+template<bool UseLLVM>
+TRuntimeNode GroupKeys(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto keyExtractor = [&](TRuntimeNode item) {
+ return item;
+ };
+
+ stream = pgmBuilder.GroupingCore(stream, groupSwitch, keyExtractor);
+ return pgmBuilder.Map(stream, [&](TRuntimeNode grpItem) {
+ return pgmBuilder.ToString(pgmBuilder.Nth(grpItem, 0));
+ });
+}
+
+template<bool UseLLVM>
+TRuntimeNode StreamToString(TSetup<UseLLVM>& setup, TRuntimeNode stream) {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ return pgmBuilder.Squeeze(stream, pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|"), [&] (TRuntimeNode item, TRuntimeNode state) {
+ return pgmBuilder.Concat(pgmBuilder.Concat(state, item), pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|"));
+ }, {}, {});
+}
+
+} // unnamed
+
+
+Y_UNIT_TEST_SUITE(TMiniKQLGroupingTest) {
+ Y_UNIT_TEST_LLVM(TestGrouping) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0));
+ });
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingKeyNotEquals) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ return pgmBuilder.NotEquals(item, key);
+ });
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*|*11*|*00*00*00*|*11*|*22*|*33*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithEmptyInput) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup, 0);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0));
+ });
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestSingleGroup) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ Y_UNUSED(item);
+ return pgmBuilder.NewDataLiteral<bool>(false);
+ });
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*01*00*00*00*01*02*03*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithYield) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
+ stream = pgmBuilder.Switch(stream,
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item1) {
+ return Group(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) {
+ Y_UNUSED(key);
+ return pgmBuilder.Equals(item2, pgmBuilder.NewDataLiteral<ui64>(0));
+ });
+ },
+ 1,
+ pgmBuilder.NewStreamType(pgmBuilder.NewDataType(NUdf::EDataSlot::String))
+ );
+
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithoutFetchingSubStreams) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+
+ stream = GroupKeys(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0));
+ });
+
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithYieldAndWithoutFetchingSubStreams) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
+ stream = pgmBuilder.Switch(stream,
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item1) {
+ return GroupKeys(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) {
+ Y_UNUSED(key);
+ return pgmBuilder.Equals(item2, pgmBuilder.NewDataLiteral<ui64>(0));
+ });
+ },
+ 1,
+ pgmBuilder.NewStreamType(pgmBuilder.NewDataType(NUdf::EDataSlot::String))
+ );
+
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithHandler) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ stream = Group(setup, stream,
+ [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0));
+ },
+ [&](TRuntimeNode item) {
+ return pgmBuilder.Add(pgmBuilder.Convert(item, pgmBuilder.NewDataType(NUdf::EDataSlot::Int32)), pgmBuilder.NewDataLiteral<ui64>(1));
+ }
+ );
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*01*|*01*02*|*01*|*01*|*01*02*03*04*|");
+ }
+}
+
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp
new file mode 100644
index 00000000000..07c933affb5
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_heap_ut.cpp
@@ -0,0 +1,319 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+#include <ydb/library/yql/utils/sort.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLHeapTest) {
+ Y_UNIT_TEST_LLVM(TestMakeHeap) {
+ const std::array<float, 10U> xxx = {{0.f, 13.f, -3.14f, 1212.f, -7898.8f, 21E4f, HUGE_VALF, -HUGE_VALF, 3673.f, -32764.f}};
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ std::array<TRuntimeNode, 10U> data;
+ std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](float f) { return pb.NewDataLiteral(f); } );
+
+ const auto type = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(type, data);
+
+ const auto pgmReturn = pb.MakeHeap(list,
+ [&](TRuntimeNode l, TRuntimeNode r) {
+ return pb.AggrLess(pb.Abs(l), pb.Abs(r));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& result = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size());
+
+ auto copy = xxx;
+ std::make_heap(copy.begin(), copy.end(), [](float l, float r){ return std::abs(l) < std::abs(r); });
+
+ for (auto i = 0U; i < copy.size(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<float>());
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestPopHeap) {
+ const std::array<double, 10U> xxx = {{0.0, 13.0, -3.140, 1212.0, -7898.8, 210000.0, 17E13, -HUGE_VAL, 3673.0, -32764.0}};
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ std::array<TRuntimeNode, 10U> data;
+ std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](double f) { return pb.NewDataLiteral(f); } );
+
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, data);
+
+ const auto comparer = [&](TRuntimeNode l, TRuntimeNode r) {
+ return pb.AggrGreater(pb.Abs(l), pb.Abs(r));
+ };
+
+ const auto pgmReturn = pb.PopHeap(pb.MakeHeap(list,comparer), comparer);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& result = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size());
+
+ auto copy = xxx;
+ const auto c = [](double l, double r){ return std::abs(l) > std::abs(r); };
+ std::make_heap(copy.begin(), copy.end(), c);
+ std::pop_heap(copy.begin(), copy.end(), c);
+
+ for (auto i = 0U; i < copy.size(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<double>());
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestSortHeap) {
+ const std::array<float, 10U> xxx = {{9E9f, -HUGE_VALF, 0.003f, 137.4f, -3.1415f, 1212.f, -7898.8f, 21E4f, 3673.f, -32764.f}};
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ std::array<TRuntimeNode, 10U> data;
+ std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](float f) { return pb.NewDataLiteral(f); } );
+
+ const auto type = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(type, data);
+
+ const auto pgmReturn = pb.SortHeap(
+ pb.MakeHeap(list,
+ [&](TRuntimeNode l, TRuntimeNode r) {
+ return pb.AggrGreater(l, r);
+ }),
+ [&](TRuntimeNode l, TRuntimeNode r) {
+ return pb.AggrGreater(l, r);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& result = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size());
+
+ auto copy = xxx;
+ std::make_heap(copy.begin(), copy.end(), std::greater<float>());
+ std::sort_heap(copy.begin(), copy.end(), std::greater<float>());
+
+ for (auto i = 0U; i < copy.size(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<float>());
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestStableSort) {
+ const std::array<double, 10U> xxx = {{9E9f, -HUGE_VALF, 0.003f, HUGE_VALF, +3.1415f, -0.003f, -7898.8f, -3.1415f, 3673.f, 0.003f}};
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ std::array<TRuntimeNode, 10U> data;
+ std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](double f) { return pb.NewDataLiteral(f); } );
+
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, data);
+
+ const auto pgmReturn = pb.StableSort(list,
+ [&](TRuntimeNode l, TRuntimeNode r) {
+ return pb.AggrGreater(pb.Abs(l), pb.Abs(r));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& result = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size());
+
+ auto copy = xxx;
+ std::stable_sort(copy.begin(), copy.end(), [](double l, double r){ return std::abs(l) > std::abs(r); });
+
+ for (auto i = 0U; i < copy.size(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<double>());
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestNthElement) {
+ const std::array<float, 10U> xxx = {{0.f, 13.f, -3.14f, 1212.f, -7898.8f, 21E4f, HUGE_VALF, -HUGE_VALF, 3673.f, -32764.f}};
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ std::array<TRuntimeNode, 10U> data;
+ std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](float f) { return pb.NewDataLiteral(f); } );
+
+ const auto type = pb.NewDataType(NUdf::TDataType<float>::Id);
+ const auto list = pb.NewList(type, data);
+ const auto n = pb.NewDataLiteral<ui64>(4U);
+
+ const auto pgmReturn = pb.NthElement(list, n,
+ [&](TRuntimeNode l, TRuntimeNode r) {
+ return pb.AggrGreater(pb.Abs(l), pb.Abs(r));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& result = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size());
+
+ auto copy = xxx;
+ NYql::FastNthElement(copy.begin(), copy.begin() + 4U, copy.end(), [](float l, float r){ return std::abs(l) > std::abs(r); });
+
+ for (auto i = 0U; i < copy.size(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<float>());
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestPartialSort) {
+ const std::array<double, 10U> xxx = {{0.0, 13.0, -3.14, 1212.0, -7898.8, 21.0E4, HUGE_VAL, -HUGE_VAL, 3673.0, -32764.0}};
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ std::array<TRuntimeNode, 10U> data;
+ std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](double f) { return pb.NewDataLiteral(f); } );
+
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, data);
+ const auto n = pb.NewDataLiteral<ui64>(6U);
+
+ const auto pgmReturn = pb.PartialSort(list, n,
+ [&](TRuntimeNode l, TRuntimeNode r) {
+ return pb.AggrLess(pb.Abs(l), pb.Abs(r));
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& result = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size());
+
+ auto copy = xxx;
+ NYql::FastPartialSort(copy.begin(), copy.begin() + 6U, copy.end(), [](double l, double r){ return std::abs(l) < std::abs(r); });
+
+ for (auto i = 0U; i < copy.size(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<double>());
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestTopN) {
+ const std::array<double, 10U> xxx = {{0.0, 13.0, -3.140, -7898.8, 210000.0, 17E13, 1212.0, -HUGE_VAL, 3673.0, -32764.0}};
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ std::array<TRuntimeNode, 10U> data;
+ std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](double f) { return pb.NewDataLiteral(f); } );
+
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, data);
+
+ const auto comparator = [&](TRuntimeNode l, TRuntimeNode r) { return pb.AggrGreater(pb.Abs(l), pb.Abs(r)); };
+
+ const auto n = 5ULL;
+
+ const auto limit = pb.NewDataLiteral<ui64>(n);
+ const auto last = pb.Decrement(limit);
+
+ const auto pgmReturn = pb.Take(pb.NthElement(pb.Fold(list, pb.NewEmptyList(type),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ const auto size = pb.Length(state);
+
+ return pb.If(pb.AggrLess(size, limit),
+ pb.If(pb.AggrLess(size, last),
+ pb.Append(state, item), pb.MakeHeap(pb.Append(state, item), comparator)),
+ pb.If(comparator(item, pb.Unwrap(pb.ToOptional(state), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)),
+ pb.PushHeap(pb.Append(pb.Take(pb.PopHeap(state, comparator), pb.Decrement(size)), item), comparator),
+ state
+ )
+ );
+ }
+ ), last, comparator), limit);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& result = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), n);
+
+ auto copy = xxx;
+
+ const auto comp = [](double l, double r){ return std::abs(l) > std::abs(r); };
+ NYql::FastNthElement(copy.begin(), copy.begin() + n - 1U, copy.end(), comp);
+ const auto mm = std::minmax_element(copy.begin(), copy.begin() + n, comp);
+
+ double min = result.GetElement(0).template Get<double>(), max = min;
+ for (auto i = 1U; i < n; ++i) {
+ const auto v = result.GetElement(i).template Get<double>();
+ min = std::min(min, v, comp);
+ max = std::max(max, v, comp);
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(*mm.first, min);
+ UNIT_ASSERT_VALUES_EQUAL(*mm.second, max);
+ }
+
+ Y_UNIT_TEST_LLVM(TestTopByNthElement) {
+ const std::array<double, 10U> xxx = {{0.0, 13.0, -3.140, -7898.8, 210000.0, 17E13, 1212.0, -HUGE_VAL, 3673.0, -32764.0}};
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ std::array<TRuntimeNode, 10U> data;
+ std::transform(xxx.cbegin(), xxx.cend(), data.begin(), [&pb](double f) { return pb.NewDataLiteral(f); } );
+
+ const auto type = pb.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pb.NewList(type, data);
+
+ const auto comparator = [&](TRuntimeNode l, TRuntimeNode r) { return pb.AggrLess(pb.Abs(l), pb.Abs(r)); };
+
+ const auto n = 5ULL;
+
+ const auto limit = pb.NewDataLiteral<ui64>(n);
+ const auto reserve = pb.ShiftLeft(limit, pb.NewDataLiteral<ui8>(1U));
+ const auto last = pb.Decrement(limit);
+
+ const auto pgmReturn = pb.Take(pb.NthElement(pb.Fold(list, pb.NewEmptyList(type),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ const auto size = pb.Length(state);
+
+ return pb.If(pb.AggrLess(size, limit),
+ pb.If(pb.AggrLess(size, last),
+ pb.Append(state, item), pb.MakeHeap(pb.Append(state, item), comparator)),
+ pb.If(comparator(item, pb.Unwrap(pb.ToOptional(state), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0)),
+ pb.If(pb.AggrLess(size, reserve),
+ pb.Append(state, item),
+ pb.Take(pb.NthElement(pb.Prepend(item, pb.Skip(state, pb.NewDataLiteral<ui64>(1U))), last, comparator), limit)
+ ),
+ state
+ )
+ );
+ }
+ ), last, comparator), limit);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& result = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), n);
+
+ auto copy = xxx;
+
+ const auto comp = [](double l, double r){ return std::abs(l) < std::abs(r); };
+ NYql::FastNthElement(copy.begin(), copy.begin() + n - 1U, copy.end(), comp);
+ const auto mm = std::minmax_element(copy.begin(), copy.begin() + n, comp);
+
+ double min = result.GetElement(0).template Get<double>(), max = min;
+ for (auto i = 1U; i < n; ++i) {
+ const auto v = result.GetElement(i).template Get<double>();
+ min = std::min(min, v, comp);
+ max = std::max(max, v, comp);
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(*mm.first, min);
+ UNIT_ASSERT_VALUES_EQUAL(*mm.second, max);
+ }
+}
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp
new file mode 100644
index 00000000000..893722216d9
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_dict_ut.cpp
@@ -0,0 +1,431 @@
+#include "mkql_computation_node_ut.h"
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLJoinDictNodeTest) {
+ Y_UNIT_TEST_LLVM(TestInner) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+ const auto dict1 = pb.ToSortedDict(list1, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Payload");
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToSortedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Payload");
+ });
+
+ const auto pgmReturn = pb.JoinDict(dict1, true, dict2, true, EJoinKind::Inner);
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeft) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+ const auto dict1 = pb.ToSortedDict(list1, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Payload");
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToSortedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Payload");
+ });
+
+ const auto pgmReturn = pb.JoinDict(dict1, true, dict2, true, EJoinKind::Left);
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNIT_ASSERT(tuple.GetElement(1));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNIT_ASSERT(tuple.GetElement(1));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNIT_ASSERT(tuple.GetElement(1));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNIT_ASSERT(tuple.GetElement(1));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestRight) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+ const auto dict1 = pb.ToSortedDict(list1, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Payload");
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToSortedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Payload");
+ });
+
+ const auto pgmReturn = pb.JoinDict(dict1, true, dict2, true, EJoinKind::Right);
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(!tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestFull) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+ const auto dict1 = pb.ToSortedDict(list1, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Payload");
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToSortedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Payload");
+ });
+
+ const auto pgmReturn = pb.JoinDict(dict1, true, dict2, true, EJoinKind::Full);
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNIT_ASSERT(tuple.GetElement(1));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNIT_ASSERT(tuple.GetElement(1));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNIT_ASSERT(tuple.GetElement(1));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNIT_ASSERT(tuple.GetElement(1));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(!tuple.GetElement(0));
+ UNIT_ASSERT(tuple.GetElement(1));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Z");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+
+ Y_UNIT_TEST_LLVM(TestInnerFlat) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1U);
+ const auto key2 = pb.NewDataLiteral<ui32>(2U);
+ const auto key3 = pb.NewDataLiteral<ui32>(3U);
+ const auto key4 = pb.NewDataLiteral<ui32>(4U);
+ const auto key5 = pb.NewDataLiteral<ui32>(5U);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("D");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("E");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("F");
+ const auto payload7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("G");
+ const auto payload8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("H");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+
+ const auto list1 = pb.NewList(structType, {
+ pb.NewStruct(structType, {{"Key", key1}, {"Payload", payload1}}),
+ pb.NewStruct(structType, {{"Key", key2}, {"Payload", payload2}}),
+ pb.NewStruct(structType, {{"Key", key3}, {"Payload", payload3}}),
+ pb.NewStruct(structType, {{"Key", key4}, {"Payload", payload4}})
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.NewStruct(structType, {{"Key", key2}, {"Payload", payload8}}),
+ pb.NewStruct(structType, {{"Key", key3}, {"Payload", payload7}}),
+ pb.NewStruct(structType, {{"Key", key4}, {"Payload", payload6}}),
+ pb.NewStruct(structType, {{"Key", key5}, {"Payload", payload5}})
+ });
+
+ const auto listList = pb.NewList(pb.NewListType(structType), {list1, list2});
+
+ const auto pgmReturn = pb.FlatMap(listList,
+ [&](TRuntimeNode left) {
+ const auto dict1 = pb.ToSortedDict(left, false,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Payload");
+ }, false, 0);
+ return pb.FlatMap(listList,
+ [&](TRuntimeNode right) {
+ const auto dict2 = pb.ToSortedDict(right, false,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Payload");
+ }, false, 0);
+ return pb.JoinDict(dict1, false, dict2, false, EJoinKind::Inner);
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto list = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 14U);
+
+ const auto iterator = list.GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "A");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "A");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "B");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "C");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "D");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "H");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "G");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "D");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "F");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "H");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "B");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "G");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "C");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "F");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "D");
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "H");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "H");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "G");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "G");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "F");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "F");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "E");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "E");
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp
new file mode 100644
index 00000000000..89e2827eb41
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_join_ut.cpp
@@ -0,0 +1,329 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLCommonJoinCoreTupleTest) {
+ Y_UNIT_TEST_LLVM(Inner) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)});
+
+ const auto list = pb.NewList(tupleType, {data1, data3, data2, data4});
+
+ const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType}));
+ const auto pgmReturn = pb.Collect(pb.CommonJoinCore(pb.ToFlow(list), EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, std::nullopt, EAnyJoinSettings::None, 3U, outputType));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(InnerOrderLeftFirst) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType}));
+ const auto pgmReturn = pb.Collect(pb.CommonJoinCore(pb.ToFlow(list), EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, {0U}, EAnyJoinSettings::None, 3U, outputType));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(InnerOrderRightFirst) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)});
+
+ const auto list = pb.NewList(tupleType, {data3, data4, data1, data2});
+
+ const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType}));
+ const auto pgmReturn = pb.Collect(pb.CommonJoinCore(pb.ToFlow(list), EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, {1U}, EAnyJoinSettings::None, 3U, outputType));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u
+Y_UNIT_TEST_SUITE(TMiniKQLCommonJoinCoreWideTest) {
+ Y_UNIT_TEST_LLVM(Inner) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)});
+
+ const auto list = pb.NewList(tupleType, {data1, data3, data2, data4});
+
+ const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType}));
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.CommonJoinCore(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U)}; }),
+ EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, std::nullopt, EAnyJoinSettings::None, 3U, outputType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(InnerOrderLeftFirst) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType}));
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.CommonJoinCore(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U)}; }),
+ EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, {0U}, EAnyJoinSettings::None, 3U, outputType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(InnerOrderRightFirst) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({optionalType, optionalType, optionalType, indexType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-1)), pb.NewDataLiteral<ui32>(0)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2)), pb.NewDataLiteral<ui32>(0)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optionalType), pb.NewOptional(pb.NewDataLiteral<i32>(-3)), pb.NewDataLiteral<ui32>(1)});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optionalType), pb.NewDataLiteral<ui32>(1)});
+
+ const auto list = pb.NewList(tupleType, {data3, data4, data1, data2});
+
+ const auto outputType = pb.NewFlowType(pb.NewMultiType({optionalType, optionalType}));
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.CommonJoinCore(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U)}; }),
+ EJoinKind::Inner, {0U, 0U}, {1U, 1U}, {}, {2U}, 0ULL, {1U}, EAnyJoinSettings::None, 3U, outputType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(ExclusionOrderLeftFirstAny) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto stringType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ const auto optStrType = pb.NewOptionalType(stringType);
+ const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({optionalType, optStrType, optStrType, indexType});
+
+ const auto value1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 1"));
+ const auto value2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 2"));
+ const auto value3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 3"));
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(optStrType), value1, pb.NewDataLiteral<ui32>(1)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewEmptyOptional(optStrType), value2, pb.NewDataLiteral<ui32>(1)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(optStrType), value3, pb.NewDataLiteral<ui32>(1)});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optStrType), pb.NewEmptyOptional(optStrType), pb.NewDataLiteral<ui32>(1)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto outputType = pb.NewFlowType(pb.NewMultiType({optStrType, optStrType}));
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("ACHTUNG MINEN!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.CommonJoinCore(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.NewOptional(pb.Unwrap(pb.Nth(item, 2U), landmine, __FILE__, __LINE__, 0)), pb.Nth(item, 3U)}; }),
+ EJoinKind::Exclusion, {1U, 0U}, {2U, 1U}, {0U}, {0U}, 0ULL, {0U}, EAnyJoinSettings::Right, 3U, outputType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(ExclusionOrderRightFirstAny) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto indexType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto stringType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ const auto optStrType = pb.NewOptionalType(stringType);
+ const auto optionalType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({optionalType, optStrType, optStrType, indexType});
+
+ const auto value1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 1"));
+ const auto value2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 2"));
+ const auto value3 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("very long value 3"));
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), value1, pb.NewEmptyOptional(optStrType), pb.NewDataLiteral<ui32>(0)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), value2, pb.NewEmptyOptional(optStrType), pb.NewDataLiteral<ui32>(0)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), value3, pb.NewEmptyOptional(optStrType), pb.NewDataLiteral<ui32>(0)});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(optStrType), pb.NewEmptyOptional(optStrType), pb.NewDataLiteral<ui32>(0)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto outputType = pb.NewFlowType(pb.NewMultiType({optStrType, optStrType}));
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("ACHTUNG MINEN!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.CommonJoinCore(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.NewOptional(pb.Unwrap(pb.Nth(item, 1U), landmine, __FILE__, __LINE__, 0)), pb.Nth(item, 2U), pb.Nth(item, 3U)}; }),
+ EJoinKind::Exclusion, {1U, 0U}, {2U, 1U}, {0U}, {0U}, 0ULL, {1U}, EAnyJoinSettings::Left, 3U, outputType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long value 1");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+#endif
+}
+}
+
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp
new file mode 100644
index 00000000000..1ba1c92bdb0
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_listfromrange_ut.cpp
@@ -0,0 +1,469 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+template<bool UseLLVM, typename T>
+TRuntimeNode MakeList(TSetup<UseLLVM>& setup, T Start, T End, i64 Step, const auto dateType) {
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto start = pb.Unwrap(pb.ToIntegral(pb.NewDataLiteral<T>(Start), dateType), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+
+ const auto end = pb.Unwrap(pb.ToIntegral(pb.NewDataLiteral<T>(End), dateType), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+
+ const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(
+ NUdf::TStringRef((const char*)&Step, sizeof(Step)));
+
+ return pb.Collect(pb.ToFlow(pb.ListFromRange(start, end, step)));
+}
+}
+
+
+Y_UNIT_TEST_SUITE(TMiniKQLListFromRangeTest) {
+ Y_UNIT_TEST_LLVM(TestCorrectDate) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui16 start = 140;
+ ui16 end = 150;
+ i64 step = 86400000000LL;
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), end - start);
+ NUdf::TUnboxedValue item;
+ for (size_t i = 0; i < end - start; i++) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), start + i);
+ }
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestCorrectDateReverse) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui16 start = 150;
+ ui16 end = 140;
+ i64 step = -86400000000LL;
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), start - end);
+ NUdf::TUnboxedValue item;
+ for (int i = 0; i < start - end; i++) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), start - i);
+ }
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestCorrectDatetime) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui32 start = 140;
+ ui32 end = 150;
+ i64 step = 1000000LL;
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Datetime, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), end - start);
+ NUdf::TUnboxedValue item;
+ for (size_t i = 0; i < end - start; i++) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), start + i);
+ }
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestCorrectTimestamp) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui64 start = 140;
+ ui64 end = 150;
+ i64 step = 1LL;
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Timestamp, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), end - start);
+ NUdf::TUnboxedValue item;
+ for (size_t i = 0; i < end - start; i++) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), start + i);
+ }
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestWrongIntervalForDate) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui16 start = 140;
+ ui16 end = 150;
+ i64 step = 86400000001LL;
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 0);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestWrongIntervalForDatetime) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui32 start = 140;
+ ui32 end = 150;
+ i64 step = 1000003LL;
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Datetime, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 0);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestWrongStartType) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto value0 = ui32(1000000);
+ const auto start = pb.NewDataLiteral<ui32>(value0);
+
+ const auto value1 = ui32(1000005);
+ const auto end = pb.NewDataLiteral<NUdf::EDataSlot::Datetime>(
+ NUdf::TStringRef((const char*)&value1, sizeof(value1)));
+
+ const auto value2 = i64(1000001LL);
+ const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(
+ NUdf::TStringRef((const char*)&value2, sizeof(value2)));
+
+ UNIT_ASSERT_EXCEPTION(pb.ListFromRange(start, end, step), yexception);
+ }
+ Y_UNIT_TEST_LLVM(TestWrongEndType) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto value0 = ui32(1000000);
+ const auto start = pb.NewDataLiteral<NUdf::EDataSlot::Datetime>(
+ NUdf::TStringRef((const char*)&value0, sizeof(value0)));
+
+ const auto value1 = ui32(1000005);
+ const auto end = pb.NewDataLiteral<ui32>(value1);
+
+ const auto value2 = i64(1000001LL);
+ const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(
+ NUdf::TStringRef((const char*)&value2, sizeof(value2)));
+
+ UNIT_ASSERT_EXCEPTION(pb.ListFromRange(start, end, step), yexception);
+ }
+ Y_UNIT_TEST_LLVM(TestWrongStepType) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto value0 = ui32(1000000);
+ const auto start = pb.NewDataLiteral<NUdf::EDataSlot::Datetime>(
+ NUdf::TStringRef((const char*)&value0, sizeof(value0)));
+
+ const auto value1 = ui32(1000005);
+ const auto end = pb.NewDataLiteral<NUdf::EDataSlot::Datetime>(
+ NUdf::TStringRef((const char*)&value1, sizeof(value1)));
+
+ const auto value2 = i64(1000001LL);
+ const auto step = pb.NewDataLiteral<ui32>(value2);
+
+ UNIT_ASSERT_EXCEPTION(pb.ListFromRange(start, end, step), yexception);
+ }
+ Y_UNIT_TEST_LLVM(TestEmptyListDate) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui16 start = 150;
+ ui16 end = 144;
+ i64 step = 86400000000LL;
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 0);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestWrongStartEndTypes) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto value0 = ui16(140);
+ const auto start = pb.NewDataLiteral<NUdf::EDataSlot::Date>(
+ NUdf::TStringRef((const char*)&value0, sizeof(value0)));
+
+ const auto value1 = ui32(140 * 60 * 60 * 24 + 5);
+ const auto end = pb.NewDataLiteral<NUdf::EDataSlot::Datetime>(
+ NUdf::TStringRef((const char*)&value1, sizeof(value1)));
+
+ const auto value2 = i64(2000000LL); // 2 Seconds
+ const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(
+ NUdf::TStringRef((const char*)&value2, sizeof(value2)));
+
+ UNIT_ASSERT_EXCEPTION(pb.ListFromRange(start, end, step), yexception);
+ }
+ Y_UNIT_TEST_LLVM(TestMinOverflowForDate) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui16 start = 4;
+ ui16 end = 0;
+ i64 step = -518400000000LL; // -6 days
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), start);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestMinOverflowForDatetime) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui32 start = 9;
+ ui32 end = 0;
+ i64 step = -10000000LL; // -10 seconds
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Datetime, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), start);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestMinOverflowForTimestamp) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui64 start = 100;
+ ui64 end = 10;
+ i64 step = -110LL; // -110 microseconds
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Timestamp, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), start);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestMaxOverflowForDate) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+ ui16 start = 100;
+ ui16 end = NYql::NUdf::MAX_DATE - 1;
+ i64 step = (NYql::NUdf::MAX_DATE - 1) * 24LL * 60 * 60 * 1000000;
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), start);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestMaxOverflowForDatetime) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ ui32 start = NYql::NUdf::MAX_DATETIME - 123;
+ ui32 end = NYql::NUdf::MAX_DATETIME - 1;
+ i64 step = (NYql::NUdf::MAX_DATETIME - 1) * 1000000LL;
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Datetime, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), start);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestMaxOverflowForTimestamp) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ ui64 start = NYql::NUdf::MAX_TIMESTAMP - 123;
+ ui64 end = NYql::NUdf::MAX_TIMESTAMP - 1;
+ i64 step = NYql::NUdf::MAX_TIMESTAMP - 1;
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Timestamp, true);
+
+ const auto dates = MakeList(setup, start, end, step, dateType);
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), start);
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+ Y_UNIT_TEST_LLVM(TestDifferentTimezonesForTzDate) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+ const auto canada = pb.NewDataLiteral<ui16>(375U);
+ const auto europe = pb.NewDataLiteral<ui16>(459U);
+ const auto value2 = i64(24LL * 60 * 60 * 1000000); // 1 Day
+ const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(
+ NUdf::TStringRef((const char*)&value2, sizeof(value2)));
+
+ const auto day1 = pb.ToIntegral(pb.NewDataLiteral<ui16>(123), dateType);
+ const auto day2 = pb.ToIntegral(pb.NewDataLiteral<ui16>(123 + 5), dateType);
+ const auto date1 = pb.Unwrap(pb.AddTimezone(day1, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto date2 = pb.Unwrap(pb.AddTimezone(day2, europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto dates = pb.ListFromRange(date1, date2, step);
+
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 5);
+ NUdf::TUnboxedValue item;
+ for (size_t i = 123; i < 123 + 5; i++) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), i);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetTimezoneId(), 375U);
+ }
+ }
+ Y_UNIT_TEST_LLVM(TestSameTimezonesForTzDate) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Date, true);
+ const auto canada = pb.NewDataLiteral<ui16>(375U);
+ const auto value2 = i64(24LL * 60 * 60 * 1000000); // 1 Day
+ const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(
+ NUdf::TStringRef((const char*)&value2, sizeof(value2)));
+
+ const auto day1 = pb.ToIntegral(pb.NewDataLiteral<ui16>(123), dateType);
+ const auto day2 = pb.ToIntegral(pb.NewDataLiteral<ui16>(123 + 5), dateType);
+ const auto date1 = pb.Unwrap(pb.AddTimezone(day1, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto date2 = pb.Unwrap(pb.AddTimezone(day2, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto dates = pb.ListFromRange(date1, date2, step);
+
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 5);
+ NUdf::TUnboxedValue item;
+ for (size_t i = 123; i < 123 + 5; i++) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui16>(), i);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetTimezoneId(), 375U);
+ }
+ }
+ Y_UNIT_TEST_LLVM(TestDifferentTimezonesForTzDatetime) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Datetime, true);
+ const auto canada = pb.NewDataLiteral<ui16>(375U);
+ const auto europe = pb.NewDataLiteral<ui16>(459U);
+ const auto value2 = i64(1000000LL); // 1 Second
+ const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(
+ NUdf::TStringRef((const char*)&value2, sizeof(value2)));
+
+ const auto day1 = pb.ToIntegral(pb.NewDataLiteral<ui32>(123), dateType);
+ const auto day2 = pb.ToIntegral(pb.NewDataLiteral<ui32>(123 + 5), dateType);
+ const auto date1 = pb.Unwrap(pb.AddTimezone(day1, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto date2 = pb.Unwrap(pb.AddTimezone(day2, europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto dates = pb.ListFromRange(date1, date2, step);
+
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 5);
+ NUdf::TUnboxedValue item;
+ for (size_t i = 123; i < 123 + 5; i++) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), i);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetTimezoneId(), 375U);
+ }
+ }
+ Y_UNIT_TEST_LLVM(TestDifferentTimezonesForTzTimestamp) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dateType = pb.NewDataType(NUdf::EDataSlot::Timestamp, true);
+ const auto europe = pb.NewDataLiteral<ui16>(459U);
+ const auto canada = pb.NewDataLiteral<ui16>(375U);
+ const auto value2 = i64(1LL); // 1 Microsecond
+ const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(
+ NUdf::TStringRef((const char*)&value2, sizeof(value2)));
+
+ const auto day1 = pb.ToIntegral(pb.NewDataLiteral<ui64>(123), dateType);
+ const auto day2 = pb.ToIntegral(pb.NewDataLiteral<ui64>(123 + 5), dateType);
+ const auto date1 = pb.Unwrap(pb.AddTimezone(day1, europe), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto date2 = pb.Unwrap(pb.AddTimezone(day2, canada), pb.NewDataLiteral<NUdf::EDataSlot::String>(""), "", 0, 0);
+ const auto dates = pb.ListFromRange(date1, date2, step);
+
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ const auto iterator = list.GetListIterator();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 5);
+ NUdf::TUnboxedValue item;
+ for (size_t i = 123; i < 123 + 5; i++) {
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), i);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetTimezoneId(), 459U);
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestResverseUnsignedShorts) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto from = pb.NewDataLiteral<ui16>(60000U);
+ const auto to = pb.NewDataLiteral<ui16>(59990U);
+ const auto step = pb.NewDataLiteral<i16>(-2);
+
+ const auto dates = pb.Collect(pb.ToFlow(pb.ListFromRange(from, to, step)));
+
+ const auto graph = setup.BuildGraph(dates);
+ const auto list = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(list.GetListLength(), 5UL);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(0U).template Get<ui16>(), 60000U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(1U).template Get<ui16>(), 59998U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(2U).template Get<ui16>(), 59996U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(3U).template Get<ui16>(), 59994U);
+ UNIT_ASSERT_VALUES_EQUAL(list.GetElement(4U).template Get<ui16>(), 59992U);
+ }
+}
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp
new file mode 100644
index 00000000000..45f039e2099
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_map_join_ut.cpp
@@ -0,0 +1,1150 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLMapJoinCoreTest) {
+ Y_UNIT_TEST_LLVM(TestInnerOnTuple) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto optionalUi64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id, true);
+ const auto tupleType = pb.NewTupleType({optionalUi64Type, optionalUi64Type});
+ const auto emptyOptionalUi64 = pb.NewEmptyOptional(optionalUi64Type);
+
+ const auto key1 = pb.NewTuple(tupleType, {
+ pb.NewOptional(pb.NewDataLiteral<ui64>(1)),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(1)),
+ });
+ const auto key2 = pb.NewTuple(tupleType, {
+ pb.NewOptional(pb.NewDataLiteral<ui64>(2)),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(2)),
+ });
+ const auto key3 = pb.NewTuple(tupleType, {
+ pb.NewOptional(pb.NewDataLiteral<ui64>(3)),
+ emptyOptionalUi64,
+ });
+ const auto key4 = pb.NewTuple(tupleType, {
+ pb.NewOptional(pb.NewDataLiteral<ui64>(4)),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(4)),
+ });
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", tupleType},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToSortedDict(list2, false,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload"));
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewStructType({
+ {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ {"Right", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::Inner, {0U}, {1U, 0U}, {0U, 1U}, resultType));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+
+ Y_UNIT_TEST_LLVM(TestInner) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(3);
+ const auto key4 = pb.NewDataLiteral<ui32>(4);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToHashedDict(list2, false,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload"));
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewStructType({
+ {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ {"Right", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::Inner, {0U}, {1U, 0U}, {0U, 1U}, resultType));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestInnerMulti) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload"));
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewStructType({
+ {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ {"Right", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::Inner, {0U}, {1U, 0U}, {0U, 1U}, resultType));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeft) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(3);
+ const auto key4 = pb.NewDataLiteral<ui32>(4);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToHashedDict(list2, false,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload"));
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewStructType({
+ {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ {"Right", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::Left, {0U}, {1U, 0U}, {0U, 1U}, resultType));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftMulti) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload"));
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewStructType({
+ {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ {"Right", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::Left, {0U}, {1U, 0U}, {0U, 1U}, resultType));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftSemi) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload"));
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewStructType({
+ {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::LeftSemi, {0U}, {1U, 1U, 0U, 0U}, {}, resultType));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 2);
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "B");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 2);
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "C");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftOnly) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload"));
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewStructType({
+ {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::LeftOnly, {0U}, {1U, 1U, 0U, 0U}, {}, resultType));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 1);
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "A");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftSemiWithNullKey) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3));
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id, true)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key0), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key0), "Payload", payload3),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload"));
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewStructType({
+ {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::LeftSemi, {0U}, {1U, 1U, 0U, 0U}, {}, resultType));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 2);
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "B");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 2);
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "C");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftOnlyWithNullKey) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3));
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto structType = pb.NewStructType({
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id, true)},
+ {"Payload", pb.NewDataType(NUdf::TDataType<char*>::Id)}
+ });
+ const auto list1 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key0), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key1), "Payload", payload1),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload2),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload3)
+ });
+
+ const auto list2 = pb.NewList(structType, {
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key0), "Payload", payload3),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key2), "Payload", payload4),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key3), "Payload", payload5),
+ pb.AddMember(pb.AddMember(pb.NewEmptyStruct(), "Key", key4), "Payload", payload6)
+ });
+
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Member(item, "Key");
+ },
+ [&](TRuntimeNode item) {
+ return pb.AddMember(pb.NewEmptyStruct(), "Payload", pb.Member(item, "Payload"));
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewStructType({
+ {"Left", pb.NewDataType(NUdf::TDataType<char*>::Id)},
+ {"Key", pb.NewDataType(NUdf::TDataType<ui32>::Id)},
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.MapJoinCore(pb.ToFlow(list1), dict2, EJoinKind::LeftOnly, {0U}, {1U, 1U, 0U, 0U}, {}, resultType));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT(!tuple.GetElement(0));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(0).Get<ui32>(), 1);
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "A");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+}
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u
+Y_UNIT_TEST_SUITE(TMiniKQLWideMapJoinCoreTest) {
+ Y_UNIT_TEST_LLVM(TestInner) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(4);
+ const auto key4 = pb.NewDataLiteral<ui32>(4);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto dict2 = pb.ToHashedDict(list2, false,
+ [&](TRuntimeNode item) {
+ return pb.Nth(item, 0U);
+ },
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Nth(item, 1U)});
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ dict2, EJoinKind::Inner, {0U}, {1U, 0U}, {0U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestInnerMulti) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Nth(item, 0U);
+ },
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Nth(item, 1U)});
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ dict2, EJoinKind::Inner, {0U}, {1U, 0U}, {0U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeft) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(3);
+ const auto key4 = pb.NewDataLiteral<ui32>(4);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto dict2 = pb.ToHashedDict(list2, false,
+ [&](TRuntimeNode item) {
+ return pb.Nth(item, 0U);
+ },
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Nth(item, 1U)});
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ dict2, EJoinKind::Left, {0U}, {1U, 0U}, {0U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftMulti) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Nth(item, 0U);
+ },
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Nth(item, 1U)});
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ dict2, EJoinKind::Left, {0U}, {1U, 0U}, {0U, 1U}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "X");
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(1), "Y");
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftSemi) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Nth(item, 0U);
+ },
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Nth(item, 1U)});
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ dict2, EJoinKind::LeftSemi, {0U}, {1U, 0U, 0U, 1U}, {}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftOnly) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key1 = pb.NewDataLiteral<ui32>(1);
+ const auto key2 = pb.NewDataLiteral<ui32>(2);
+ const auto key3 = pb.NewDataLiteral<ui32>(2);
+ const auto key4 = pb.NewDataLiteral<ui32>(3);
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Nth(item, 0U);
+ },
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Nth(item, 1U)});
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ dict2, EJoinKind::LeftOnly, {0U}, {1U, 0U, 0U, 1U}, {}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 1);
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftSemiWithNullKey) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3));
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id, true),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload4}),
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload3}),
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Nth(item, 0U);
+ },
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Nth(item, 1U)});
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ dict2, EJoinKind::LeftSemi, {0U}, {1U, 0U, 0U, 1U}, {}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "B");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "C");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 2);
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestLeftOnlyWithNullKey) {
+ for (ui32 pass = 0; pass < 1; ++pass) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewDataType(NUdf::TDataType<ui32>::Id, true));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<ui32>(1));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key3 = pb.NewOptional(pb.NewDataLiteral<ui32>(2));
+ const auto key4 = pb.NewOptional(pb.NewDataLiteral<ui32>(3));
+ const auto payload1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("A");
+ const auto payload2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("B");
+ const auto payload3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("C");
+ const auto payload4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("X");
+ const auto payload5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Y");
+ const auto payload6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("Z");
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id, true),
+ pb.NewDataType(NUdf::TDataType<char*>::Id)
+ });
+
+ const auto list1 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload4}),
+ pb.NewTuple({key1, payload1}),
+ pb.NewTuple({key2, payload2}),
+ pb.NewTuple({key3, payload3})
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({key0, payload3}),
+ pb.NewTuple({key2, payload4}),
+ pb.NewTuple({key3, payload5}),
+ pb.NewTuple({key4, payload6})
+ });
+
+ const auto dict2 = pb.ToHashedDict(list2, true,
+ [&](TRuntimeNode item) {
+ return pb.Nth(item, 0U);
+ },
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Nth(item, 1U)});
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<ui32>::Id)
+ }));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.MapJoinCore(pb.ExpandMap(pb.ToFlow(list1),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ dict2, EJoinKind::LeftOnly, {0U}, {1U, 0U, 0U, 1U}, {}, resultType),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); })
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue tuple;
+
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "X");
+ UNIT_ASSERT(!tuple.GetElement(1));
+ UNIT_ASSERT(iterator.Next(tuple));
+ UNBOXED_VALUE_STR_EQUAL(tuple.GetElement(0), "A");
+ UNIT_ASSERT_VALUES_EQUAL(tuple.GetElement(1).Get<ui32>(), 1);
+ UNIT_ASSERT(!iterator.Next(tuple));
+ UNIT_ASSERT(!iterator.Next(tuple));
+ }
+ }
+}
+#endif
+}
+}
+
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp
new file mode 100644
index 00000000000..a416803e447
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_mapnext_ut.cpp
@@ -0,0 +1,172 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLMapNextTest) {
+ Y_UNIT_TEST_LLVM(OverStream) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui16>(10);
+ const auto data2 = pb.NewDataLiteral<ui16>(20);
+ const auto data3 = pb.NewDataLiteral<ui16>(30);
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id);
+ const auto optDataType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, optDataType});
+
+ const auto list = pb.NewList(dataType, {data1, data2, data3});
+ const auto pgmReturn = pb.MapNext(pb.Iterator(list, {}),
+ [&](TRuntimeNode item, TRuntimeNode nextItem) {
+ return pb.NewTuple(tupleType, {item, nextItem});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 10);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui16>(), 20);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 20);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui16>(), 30);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 30);
+ UNIT_ASSERT(!item.GetElement(1).HasValue());
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(OverSingleElementStream) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui16>(10);
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id);
+ const auto optDataType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, optDataType});
+
+ const auto list = pb.NewList(dataType, {data1});
+ const auto pgmReturn = pb.MapNext(pb.Iterator(list, {}),
+ [&](TRuntimeNode item, TRuntimeNode nextItem) {
+ return pb.NewTuple(tupleType, {item, nextItem});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 10);
+ UNIT_ASSERT(!item.GetElement(1).HasValue());
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(OverEmptyStream) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id);
+ const auto optDataType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, optDataType});
+
+ const auto list = pb.NewList(dataType, {});
+ const auto pgmReturn = pb.MapNext(pb.Iterator(list, {}),
+ [&](TRuntimeNode item, TRuntimeNode nextItem) {
+ return pb.NewTuple(tupleType, {item, nextItem});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(OverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui16>(10);
+ const auto data2 = pb.NewDataLiteral<ui16>(20);
+ const auto data3 = pb.NewDataLiteral<ui16>(30);
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id);
+ const auto optDataType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, optDataType});
+
+ const auto list = pb.NewList(dataType, {data1, data2, data3});
+ const auto pgmReturn = pb.FromFlow(pb.MapNext(pb.ToFlow(pb.Iterator(list, {})),
+ [&](TRuntimeNode item, TRuntimeNode nextItem) {
+ return pb.NewTuple(tupleType, {item, nextItem});
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 10);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui16>(), 20);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 20);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui16>(), 30);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 30);
+ UNIT_ASSERT(!item.GetElement(1).HasValue());
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(OverSingleElementFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui16>(10);
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id);
+ const auto optDataType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, optDataType});
+
+ const auto list = pb.NewList(dataType, {data1});
+ const auto pgmReturn = pb.FromFlow(pb.MapNext(pb.ToFlow(pb.Iterator(list, {})),
+ [&](TRuntimeNode item, TRuntimeNode nextItem) {
+ return pb.NewTuple(tupleType, {item, nextItem});
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui16>(), 10);
+ UNIT_ASSERT(!item.GetElement(1).HasValue());
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(OverEmptyFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui16>::Id);
+ const auto optDataType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, optDataType});
+
+ const auto list = pb.NewList(dataType, {});
+ const auto pgmReturn = pb.FromFlow(pb.MapNext(pb.ToFlow(pb.Iterator(list, {})),
+ [&](TRuntimeNode item, TRuntimeNode nextItem) {
+ return pb.NewTuple(tupleType, {item, nextItem});
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp
new file mode 100644
index 00000000000..d7c4e8517ba
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_saveload_ut.cpp
@@ -0,0 +1,305 @@
+#include "../mkql_multihopping.h"
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+ TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
+ return CreateDeterministicRandomProvider(1);
+ }
+
+ TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
+ return CreateDeterministicTimeProvider(10000000);
+ }
+
+ TComputationNodeFactory GetAuxCallableFactory(TWatermark& watermark) {
+ return [&watermark](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+ if (callable.GetType()->GetName() == "OneYieldStream") {
+ return new TExternalComputationNode(ctx.Mutables);
+ } else if (callable.GetType()->GetName() == "MultiHoppingCore") {
+ return WrapMultiHoppingCore(callable, ctx, watermark);
+ }
+
+ return GetBuiltinFactory()(callable, ctx);
+ };
+ }
+
+ struct TSetup {
+ TSetup(TScopedAlloc& alloc)
+ : Alloc(alloc)
+ {
+ FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ RandomProvider = CreateRandomProvider();
+ TimeProvider = CreateTimeProvider();
+
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
+ THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
+ Explorer.Walk(pgm.GetNode(), *Env);
+ TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(Watermark),
+ FunctionRegistry.Get(),
+ NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi);
+ Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
+ TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
+ return Pattern->Clone(compOpts);
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+
+ TScopedAlloc& Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
+ IComputationPattern::TPtr Pattern;
+ TWatermark Watermark;
+ };
+
+ struct TStreamWithYield : public NUdf::TBoxedValue {
+ TStreamWithYield(const TUnboxedValueVector& items, ui32 yieldPos, ui32 index)
+ : Items(items)
+ , YieldPos(yieldPos)
+ , Index(index)
+ {}
+
+ private:
+ TUnboxedValueVector Items;
+ ui32 YieldPos;
+ ui32 Index;
+
+ ui32 GetTraverseCount() const override {
+ return 0;
+ }
+
+ NUdf::TUnboxedValue Save() const override {
+ return NUdf::TUnboxedValue::Zero();
+ }
+
+ void Load(const NUdf::TStringRef& state) override {
+ Y_UNUSED(state);
+ }
+
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final {
+ if (Index >= Items.size()) {
+ return NUdf::EFetchStatus::Finish;
+ }
+ if (Index == YieldPos) {
+ return NUdf::EFetchStatus::Yield;
+ }
+ result = Items[Index++];
+ return NUdf::EFetchStatus::Ok;
+ }
+ };
+
+ THolder<IComputationGraph> BuildGraph(TSetup& setup, const std::vector<std::tuple<ui32, i64, ui32>> items,
+ ui32 yieldPos, ui32 startIndex, bool dataWatermarks) {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto structType = pgmBuilder.NewEmptyStructType();
+ structType = pgmBuilder.NewStructType(structType, "key",
+ pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ structType = pgmBuilder.NewStructType(structType, "time",
+ pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id));
+ structType = pgmBuilder.NewStructType(structType, "sum",
+ pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key");
+ auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time");
+ auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum");
+
+ auto inStreamType = pgmBuilder.NewStreamType(structType);
+
+ TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType);
+ auto streamNode = inStream.Build();
+
+ ui64 hop = 10, interval = 30, delay = 20;
+
+ auto pgmReturn = pgmBuilder.MultiHoppingCore(
+ TRuntimeNode(streamNode, false),
+ [&](TRuntimeNode item) { // keyExtractor
+ return pgmBuilder.Member(item, "key");
+ },
+ [&](TRuntimeNode item) { // timeExtractor
+ return pgmBuilder.Member(item, "time");
+ },
+ [&](TRuntimeNode item) { // init
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", pgmBuilder.Member(item, "sum"));
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) { // update
+ auto add = pgmBuilder.AggrAdd(
+ pgmBuilder.Member(item, "sum"),
+ pgmBuilder.Member(state, "sum"));
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", add);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state) { // save
+ return pgmBuilder.Member(state, "sum");
+ },
+ [&](TRuntimeNode savedState) { // load
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", savedState);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state1, TRuntimeNode state2) { // merge
+ auto add = pgmBuilder.AggrAdd(
+ pgmBuilder.Member(state1, "sum"),
+ pgmBuilder.Member(state2, "sum"));
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", add);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish
+ Y_UNUSED(time);
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("key", key);
+ members.emplace_back("sum", pgmBuilder.Member(state, "sum"));
+ return pgmBuilder.NewStruct(members);
+ },
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay
+ pgmBuilder.NewDataLiteral<bool>(dataWatermarks), // dataWatermarks
+ pgmBuilder.NewDataLiteral<bool>(false)
+ );
+
+ auto graph = setup.BuildGraph(pgmReturn, {streamNode});
+
+ TUnboxedValueVector streamItems;
+ for (size_t i = 0; i < items.size(); ++i) {
+ NUdf::TUnboxedValue* itemsPtr;
+ auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr);
+ itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(std::get<0>(items[i]));
+ itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(std::get<1>(items[i]));
+ itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(std::get<2>(items[i]));
+ streamItems.push_back(std::move(structValues));
+ }
+
+ auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
+ return graph;
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingSaveLoadTest) {
+ void TestWithSaveLoadImpl(
+ const std::vector<std::tuple<ui32, i64, ui32>> input,
+ const std::vector<std::tuple<ui32, ui32>> expected,
+ bool withTraverse,
+ bool dataWatermarks)
+ {
+ TScopedAlloc alloc(__LOCATION__);
+
+ for (ui32 yieldPos = 0; yieldPos < input.size(); ++yieldPos) {
+ std::vector<std::tuple<ui32, ui32>> result;
+
+ TSetup setup1(alloc);
+ auto graph1 = BuildGraph(setup1, input, yieldPos, 0, dataWatermarks);
+ auto root1 = graph1->GetValue();
+
+ NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
+ while (status == NUdf::EFetchStatus::Ok) {
+ NUdf::TUnboxedValue val;
+ status = root1.Fetch(val);
+ if (status == NUdf::EFetchStatus::Ok) {
+ result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>());
+ }
+ }
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
+
+ TString graphState;
+ if (withTraverse) {
+ SaveGraphState(&root1, 1, 0ULL, graphState);
+ } else {
+ graphState = graph1->SaveGraphState();
+ }
+
+ TSetup setup2(alloc);
+ auto graph2 = BuildGraph(setup2, input, -1, yieldPos, dataWatermarks);
+ NUdf::TUnboxedValue root2;
+ if (withTraverse) {
+ root2 = graph2->GetValue();
+ LoadGraphState(&root2, 1, 0ULL, graphState);
+ } else {
+ graph2->LoadGraphState(graphState);
+ root2 = graph2->GetValue();
+ }
+
+ status = NUdf::EFetchStatus::Ok;
+ while (status == NUdf::EFetchStatus::Ok) {
+ NUdf::TUnboxedValue val;
+ status = root2.Fetch(val);
+ if (status == NUdf::EFetchStatus::Ok) {
+ result.emplace_back(val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>());
+ }
+ }
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
+
+ auto sortedExpected = expected;
+ std::sort(result.begin(), result.end());
+ std::sort(sortedExpected.begin(), sortedExpected.end());
+ UNIT_ASSERT_EQUAL(result, sortedExpected);
+ }
+ }
+
+ const std::vector<std::tuple<ui32, i64, ui32>> input1 = {
+ // Group; Time; Value
+ {2, 1, 2},
+ {1, 1, 2},
+ {2, 2, 3},
+ {1, 2, 3},
+ {2, 15, 4},
+ {1, 15, 4},
+ {2, 23, 6},
+ {1, 23, 6},
+ {2, 24, 5},
+ {1, 24, 5},
+ {2, 25, 7},
+ {1, 25, 7},
+ {2, 40, 2},
+ {1, 40, 2},
+ {2, 47, 1},
+ {1, 47, 1},
+ {2, 51, 6},
+ {1, 51, 6},
+ {2, 59, 2},
+ {1, 59, 2},
+ {2, 85, 8},
+ {1, 85, 8}
+ };
+
+ const std::vector<std::tuple<ui32, ui32>> expected = {
+ {1, 8}, {1, 8}, {1, 8}, {1, 8},
+ {1, 11}, {1, 11}, {1, 21}, {1, 22},
+ {1, 27},
+ {2, 8}, {2, 8}, {2, 8}, {2, 8},
+ {2, 11}, {2, 11}, {2, 21},
+ {2, 22}, {2, 27}};
+
+ Y_UNIT_TEST(Test1) {
+ TestWithSaveLoadImpl(input1, expected, true, false);
+ }
+
+ Y_UNIT_TEST(Test2) {
+ TestWithSaveLoadImpl(input1, expected, false, false);
+ }
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp
new file mode 100644
index 00000000000..b47cfc222ef
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multihopping_ut.cpp
@@ -0,0 +1,626 @@
+#include "../mkql_multihopping.h"
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+ struct TInputItem {
+ ui32 Key = 0;
+ i64 Time = 0;
+ ui32 Val = 0;
+ };
+
+ struct TOutputItem {
+ ui32 Key = 0;
+ ui32 Val = 0;
+ ui64 Time = 0;
+
+ constexpr bool operator==(const TOutputItem& rhs) const
+ {
+ return this->Key == rhs.Key && this->Val == rhs.Val && this->Time == rhs.Time;
+ }
+ };
+
+ struct TOutputGroup {
+ TOutputGroup(std::initializer_list<TOutputItem> items) : Items(items) {}
+
+ std::vector<TOutputItem> Items;
+ };
+
+ std::vector<TOutputItem> Ordered(std::vector<TOutputItem> vec) {
+ std::sort(vec.begin(), vec.end(), [](auto l, auto r) {
+ return std::make_tuple(l.Key, l.Val, l.Time) < std::make_tuple(r.Key, r.Val, r.Time);
+ });
+ return vec;
+ }
+
+ IOutputStream &operator<<(IOutputStream &output, std::vector<TOutputItem> items) {
+ output << "[";
+ for (ui32 i = 0; i < items.size(); ++i) {
+ output << "(" << items.at(i).Key << ";" << items.at(i).Val << ";" << items.at(i).Time << ")";
+ if (i != items.size() - 1)
+ output << ",";
+ }
+ output << "]";
+ return output;
+ }
+
+ TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
+ return CreateDeterministicRandomProvider(1);
+ }
+
+ TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
+ return CreateDeterministicTimeProvider(10000000);
+ }
+
+ TComputationNodeFactory GetAuxCallableFactory(TWatermark& watermark) {
+ return [&watermark](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+ if (callable.GetType()->GetName() == "MyStream") {
+ return new TExternalComputationNode(ctx.Mutables);
+ } else if (callable.GetType()->GetName() == "MultiHoppingCore") {
+ return WrapMultiHoppingCore(callable, ctx, watermark);
+ }
+
+ return GetBuiltinFactory()(callable, ctx);
+ };
+ }
+
+ struct TSetup {
+ TSetup(TScopedAlloc& alloc, TWatermark& watermark, bool watermarkMode = false)
+ : FunctionRegistry(CreateFunctionRegistry(CreateBuiltinRegistry()))
+ , RandomProvider(CreateRandomProvider())
+ , TimeProvider(CreateTimeProvider())
+ , StatsResgistry(CreateDefaultStatsRegistry())
+ , Alloc(alloc)
+ , Watermark(watermark)
+ , WatermarkMode(watermarkMode)
+ {
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
+ THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
+ Explorer.Walk(pgm.GetNode(), *Env);
+ TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(Watermark),
+ FunctionRegistry.Get(),
+ NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi,
+ StatsResgistry.Get());
+ Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
+ TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
+ return Pattern->Clone(compOpts);
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+ IStatsRegistryPtr StatsResgistry;
+
+ TScopedAlloc& Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
+ IComputationPattern::TPtr Pattern;
+
+ TWatermark& Watermark;
+ bool WatermarkMode;
+ };
+
+ struct TStream : public NUdf::TBoxedValue {
+ TStream(const TUnboxedValueVector& items, std::function<void()> fetchCallback, bool* yield)
+ : Items(items)
+ , FetchCallback(fetchCallback)
+ , yield(yield) {}
+
+ private:
+ TUnboxedValueVector Items;
+ ui32 Index = 0;
+ std::function<void()> FetchCallback;
+ bool* yield;
+
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final {
+ FetchCallback();
+ if (*yield) {
+ return NUdf::EFetchStatus::Yield;
+ }
+ if (Index >= Items.size()) {
+ return NUdf::EFetchStatus::Finish;
+ }
+ result = Items[Index++];
+ return NUdf::EFetchStatus::Ok;
+ }
+ };
+
+ THolder<IComputationGraph> BuildGraph(
+ TSetup& setup,
+ const std::vector<TInputItem> items,
+ std::function<void()> fetchCallback,
+ bool dataWatermarks,
+ bool* yield,
+ ui64 hop = 10,
+ ui64 interval = 30,
+ ui64 delay = 20)
+ {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto structType = pgmBuilder.NewEmptyStructType();
+ structType = pgmBuilder.NewStructType(structType, "key",
+ pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ structType = pgmBuilder.NewStructType(structType, "time",
+ pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id));
+ structType = pgmBuilder.NewStructType(structType, "sum",
+ pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ auto keyIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("key");
+ auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time");
+ auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum");
+
+ auto inStreamType = pgmBuilder.NewStreamType(structType);
+
+ TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "MyStream", inStreamType);
+ auto streamNode = inStream.Build();
+
+ auto pgmReturn = pgmBuilder.MultiHoppingCore(
+ TRuntimeNode(streamNode, false),
+ [&](TRuntimeNode item) { // keyExtractor
+ return pgmBuilder.Member(item, "key");
+ },
+ [&](TRuntimeNode item) { // timeExtractor
+ return pgmBuilder.Member(item, "time");
+ },
+ [&](TRuntimeNode item) { // init
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", pgmBuilder.Member(item, "sum"));
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) { // update
+ auto add = pgmBuilder.AggrAdd(
+ pgmBuilder.Member(item, "sum"),
+ pgmBuilder.Member(state, "sum"));
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", add);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state) { // save
+ return pgmBuilder.Member(state, "sum");
+ },
+ [&](TRuntimeNode savedState) { // load
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", savedState);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state1, TRuntimeNode state2) { // merge
+ auto add = pgmBuilder.AggrAdd(
+ pgmBuilder.Member(state1, "sum"),
+ pgmBuilder.Member(state2, "sum"));
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", add);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode key, TRuntimeNode state, TRuntimeNode time) { // finish
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("key", key);
+ members.emplace_back("sum", pgmBuilder.Member(state, "sum"));
+ members.emplace_back("time", time);
+ return pgmBuilder.NewStruct(members);
+ },
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))), // delay
+ pgmBuilder.NewDataLiteral<bool>(dataWatermarks),
+ pgmBuilder.NewDataLiteral<bool>(setup.WatermarkMode)
+ );
+
+ auto graph = setup.BuildGraph(pgmReturn, {streamNode});
+
+ TUnboxedValueVector streamItems;
+ for (size_t i = 0; i < items.size(); ++i) {
+ NUdf::TUnboxedValue* itemsPtr;
+ auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(3, itemsPtr);
+ itemsPtr[keyIndex] = NUdf::TUnboxedValuePod(items.at(i).Key);
+ itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items.at(i).Time);
+ itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items.at(i).Val);
+ streamItems.push_back(std::move(structValues));
+ }
+
+ auto streamValue = NUdf::TUnboxedValuePod(new TStream(streamItems, fetchCallback, yield));
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
+ return graph;
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLMultiHoppingTest) {
+ void TestImpl(
+ const std::vector<TInputItem>& input,
+ const std::vector<TOutputGroup>& expected,
+ bool dataWatermarks,
+ ui64 hop = 10,
+ ui64 interval = 30,
+ ui64 delay = 20,
+ std::function<void(ui32, TSetup&)> customCheck = [](ui32, TSetup&){},
+ TWatermark* watermark = nullptr,
+ bool* yield = nullptr,
+ std::function<void()> fetch_callback= [](){},
+ bool watermarkMode = false)
+ {
+ bool yield_clone = false;
+ if (!yield) {
+ yield = &yield_clone;
+ }
+ if (watermarkMode) {
+ dataWatermarks = false;
+ }
+ TWatermark watermark_clone{TInstant::Zero()};
+ if (watermark == nullptr) {
+ watermark = &watermark_clone;
+ }
+ TScopedAlloc alloc(__LOCATION__);
+ TSetup setup1(alloc, *watermark, watermarkMode);
+
+ ui32 curGroupId = 0;
+ std::vector<TOutputItem> curResult;
+
+ auto check = [&curResult, &curGroupId, &expected, customCheck, &setup1, &fetch_callback]() {
+ fetch_callback();
+ auto expectedItems = Ordered(expected.at(curGroupId).Items); // Add more empty lists at yield in expected
+ curResult = Ordered(curResult);
+ UNIT_ASSERT_EQUAL_C(curResult, expectedItems, "curGroup: " << curGroupId << " actual: " << curResult << " expected: " << expectedItems);
+ customCheck(curGroupId, setup1);
+ curGroupId++;
+ curResult.clear();
+ };
+
+ auto graph1 = BuildGraph(setup1, input, check, dataWatermarks, yield, hop, interval, delay);
+
+ auto root1 = graph1->GetValue();
+
+ NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
+ while (status == NUdf::EFetchStatus::Ok || status == NUdf::EFetchStatus::Yield) {
+ NUdf::TUnboxedValue val;
+ status = root1.Fetch(val);
+ if (status == NUdf::EFetchStatus::Ok) {
+ curResult.emplace_back(TOutputItem{val.GetElement(0).Get<ui32>(), val.GetElement(1).Get<ui32>(), val.GetElement(2).Get<ui64>()});
+ }
+ }
+
+ check();
+ UNIT_ASSERT_EQUAL_C(curGroupId, expected.size(), "1: " << curGroupId << " 2: " << expected.size());
+ }
+
+ void TestWatermarksImpl(
+ const std::vector<TInputItem>& input,
+ const std::vector<TOutputGroup>& expected,
+ const std::vector<std::pair<ui64, TInstant>>& watermarks)
+ {
+ bool yield = false;
+ TWatermark watermark;
+ ui64 inp_index = 0;
+ ui64 pattern_index = 0;
+ auto avant_fetch = [&yield, &watermark, &watermarks, &inp_index, &pattern_index](){
+ yield = false;
+ if (pattern_index >= watermarks.size()) {
+ return;
+ }
+ if (inp_index == watermarks[pattern_index].first) {
+ yield = true;
+ watermark.WatermarkIn = watermarks[pattern_index].second;
+ ++pattern_index;
+ } else {
+ ++inp_index;
+ }
+ };
+ TestImpl(input, expected, false, 10, 30, 20, [](ui32, TSetup&){}, &watermark, &yield, avant_fetch, true);
+ }
+
+ Y_UNIT_TEST(TestThrowWatermarkFromPast) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 101, 2},
+ {1, 131, 3},
+ {1, 200, 4},
+ {1, 300, 5},
+ {1, 400, 6}
+ };
+
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({})
+ };
+ std::vector<std::pair<ui64, TInstant>> yield_pattern = {
+ {2, TInstant::MicroSeconds(20)},
+ {3, TInstant::MicroSeconds(40)}
+ };
+ TestWatermarksImpl(input, expected, yield_pattern);
+ }
+
+ Y_UNIT_TEST(TestThrowWatermarkFromFuture) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 101, 2},
+ {1, 131, 3},
+ {1, 200, 4},
+ {1, 300, 5},
+ {1, 400, 6}
+ };
+
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({})
+ };
+ std::vector<std::pair<ui64, TInstant>> yield_pattern = {
+ {2, TInstant::MicroSeconds(1000)},
+ {3, TInstant::MicroSeconds(2000)}
+ };
+ TestWatermarksImpl(input, expected, yield_pattern);
+ }
+
+ Y_UNIT_TEST(TestWatermarkFlow1) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 101, 2},
+ {1, 131, 3},
+ {1, 200, 4},
+ {1, 300, 5},
+ {1, 400, 6}
+ };
+
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 2, 110},{1, 2, 120},{1, 2, 130}}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({})
+ };
+ std::vector<std::pair<ui64, TInstant>> yield_pattern = {
+ {0, TInstant::MicroSeconds(100)},
+ {3, TInstant::MicroSeconds(200)}
+ };
+ TestWatermarksImpl(input, expected, yield_pattern);
+ }
+
+ Y_UNIT_TEST(TestWatermarkFlow2) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 100, 2},
+ {1, 105, 3},
+ {1, 80, 4},
+ {1, 107, 5},
+ {1, 106, 6}
+ };
+
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 4, 90}, {1, 4, 100}, {1, 4, 110}})
+ };
+ std::vector<std::pair<ui64, TInstant>> yield_pattern = {
+ {0, TInstant::MicroSeconds(76)},
+ };
+ TestWatermarksImpl(input, expected, yield_pattern);
+ }
+
+ Y_UNIT_TEST(TestWatermarkFlow3) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 90, 2},
+ {1, 99, 3},
+ {1, 80, 4},
+ {1, 107, 5},
+ {1, 106, 6}
+ };
+
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 4, 90}, {1, 9, 100}, {1, 9, 110}, {1, 5, 120}})
+ };
+ std::vector<std::pair<ui64, TInstant>> yield_pattern = {
+ {0, TInstant::MicroSeconds(76)},
+ };
+ TestWatermarksImpl(input, expected, yield_pattern);
+ }
+
+ Y_UNIT_TEST(TestDataWatermarks) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 101, 2},
+ {2, 101, 2},
+ {1, 111, 3},
+ {2, 140, 5},
+ {2, 160, 1}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 2, 110}, {1, 5, 120}, {2, 2, 110}, {2, 2, 120}}),
+ TOutputGroup({{2, 2, 130}, {1, 5, 130}, {1, 3, 140}}),
+ TOutputGroup({{2, 5, 150}, {2, 5, 160}, {2, 6, 170}, {2, 1, 180}, {2, 1, 190}}),
+ };
+ TestImpl(input, expected, true);
+ }
+
+ Y_UNIT_TEST(TestDataWatermarksNoGarbage) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 100, 2},
+ {2, 150, 1}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 2, 110}, {1, 2, 120}, {1, 2, 130}}),
+ TOutputGroup({{2, 1, 160}, {2, 1, 170}, {2, 1, 180}}),
+ };
+ TestImpl(input, expected, true, 10, 30, 20,
+ [](ui32 curGroup, TSetup& setup) {
+ if (curGroup != 2) {
+ return;
+ }
+
+ setup.StatsResgistry->ForEachStat([](const TStatKey& key, i64 value) {
+ if (key.GetName() == "MultiHop_KeysCount") {
+ UNIT_ASSERT_EQUAL_C(value, 1, "actual: " << value << " expected: " << 1);
+ }
+ });
+ });
+ }
+
+ Y_UNIT_TEST(TestValidness1) {
+ const std::vector<TInputItem> input1 = {
+ // Group; Time; Value
+ {1, 101, 2},
+ {2, 101, 2},
+ {1, 111, 3},
+ {2, 140, 5},
+ {2, 160, 1}
+ };
+
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{2, 2, 110}, {2, 2, 120}}),
+ TOutputGroup({{2, 2, 130}}),
+ TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}, {2, 5, 150},
+ {2, 5, 160}, {2, 6, 170}, {2, 1, 190}, {2, 1, 180}}),
+ };
+ TestImpl(input1, expected, false);
+ }
+
+ Y_UNIT_TEST(TestValidness2) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {2, 101, 2}, {1, 101, 2}, {2, 102, 3}, {1, 102, 3}, {2, 115, 4},
+ {1, 115, 4}, {2, 123, 6}, {1, 123, 6}, {2, 124, 5}, {1, 124, 5},
+ {2, 125, 7}, {1, 125, 7}, {2, 140, 2}, {1, 140, 2}, {2, 147, 1},
+ {1, 147, 1}, {2, 151, 6}, {1, 151, 6}, {2, 159, 2}, {1, 159, 2},
+ {2, 185, 8}, {1, 185, 8}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({{1, 5, 110}, {1, 9, 120}, {2, 5, 110}, {2, 9, 120}}),
+ TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({{2, 27, 130}, {1, 27, 130}}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({{2, 22, 140}, {2, 21, 150}, {2, 11, 160}, {1, 22, 140}, {1, 21, 150}, {1, 11, 160}}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 11, 170}, {1, 8, 180}, {1, 8, 190}, {1, 8, 200}, {1, 8, 210}, {2, 11, 170},
+ {2, 8, 180}, {2, 8, 190}, {2, 8, 200}, {2, 8, 210}}),
+ };
+
+ TestImpl(input, expected, true);
+ }
+
+ Y_UNIT_TEST(TestValidness3) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 105, 1}, {1, 107, 4}, {2, 106, 3}, {1, 111, 7}, {1, 117, 3},
+ {2, 110, 2}, {1, 108, 9}, {1, 121, 4}, {2, 107, 2}, {2, 141, 5},
+ {1, 141, 10}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({{1, 14, 110}, {2, 3, 110}}),
+ TOutputGroup({}),
+ TOutputGroup({{2, 7, 115}, {2, 2, 120}, {1, 21, 115}, {1, 10, 120}, {1, 7, 125}, {1, 4, 130}}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 10, 145}, {1, 10, 150}, {2, 5, 145}, {2, 5, 150}})
+ };
+
+ TestImpl(input, expected, true, 5, 10, 10);
+ }
+
+ Y_UNIT_TEST(TestDelay) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 101, 3}, {1, 111, 5}, {1, 120, 7}, {1, 80, 9}, {1, 79, 11}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({}), TOutputGroup({}),
+ TOutputGroup({{1, 12, 110}, {1, 8, 120}, {1, 15, 130}, {1, 12, 140}, {1, 7, 150}})
+ };
+
+ TestImpl(input, expected, false);
+ }
+
+ Y_UNIT_TEST(TestWindowsBeforeFirstElement) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 101, 2}, {1, 111, 3}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 2, 110}, {1, 5, 120}, {1, 5, 130}, {1, 3, 140}})
+ };
+
+ TestImpl(input, expected, false);
+ }
+
+ Y_UNIT_TEST(TestSubzeroValues) {
+ const std::vector<TInputItem> input = {
+ // Group; Time; Value
+ {1, 1, 2}
+ };
+ const std::vector<TOutputGroup> expected = {
+ TOutputGroup({}),
+ TOutputGroup({}),
+ TOutputGroup({{1, 2, 30}}),
+ };
+
+ TestImpl(input, expected, false);
+ }
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp
new file mode 100644
index 00000000000..0c88e86c046
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_multimap_ut.cpp
@@ -0,0 +1,171 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLMultiMapTest) {
+ Y_UNIT_TEST_LLVM(TestOverList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto data3 = pb.NewDataLiteral<ui32>(3);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3});
+ const auto pgmReturn = pb.MultiMap(list,
+ [&](TRuntimeNode item) {
+ return TRuntimeNode::TList{pb.Add(item, data1), item, pb.Mul(item, data2)};
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 6);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverLazyList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto data3 = pb.NewDataLiteral<ui32>(3);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3});
+ const auto pgmReturn = pb.MultiMap(pb.LazyList(list),
+ [&](TRuntimeNode item) {
+ return TRuntimeNode::TList{pb.Add(item, data1), item, pb.Mul(item, data2)};
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 6);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestOverFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<ui32>(2);
+ const auto data3 = pb.NewDataLiteral<ui32>(3);
+ const auto dataType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto list = pb.NewList(dataType, {data1, data2, data3});
+ const auto pgmReturn = pb.Collect(pb.MultiMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) {
+ return TRuntimeNode::TList{pb.Add(item, data1), item, pb.Mul(item, data2)};
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 6);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u
+ Y_UNIT_TEST_LLVM(TestFlattenByNarrow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMultiMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items[2U], items[1U], items[0U] }; }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 3);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+#endif
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp
new file mode 100644
index 00000000000..91cf6e3c77e
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_rh_hash_ut.cpp
@@ -0,0 +1,271 @@
+#include "../mkql_rh_hash.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <unordered_map>
+#include <unordered_set>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLRobinHoodHashTest) {
+ Y_UNIT_TEST(Map) {
+ TRobinHoodHashMap<i32> rh(sizeof(i64));
+ std::unordered_map<i32, i64> h;
+ for (ui64 i = 0; i < 10000; ++i) {
+ auto k = i % 1000;
+ auto [it, inserted] = h.emplace(k, 0);
+ bool isNew;
+ auto iter = rh.Insert(k, isNew);
+ UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), k);
+ UNIT_ASSERT_VALUES_EQUAL(isNew, inserted);
+ it->second += i;
+ if (isNew) {
+ *(i64*)rh.GetMutablePayload(iter) = i;
+ rh.CheckGrow();
+ } else {
+ *(i64*)rh.GetMutablePayload(iter) += i;
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize());
+ }
+
+ for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) {
+ if (!rh.IsValid(it)) {
+ continue;
+ }
+
+ auto key = rh.GetKey(it);
+ auto hit = h.find(key);
+ UNIT_ASSERT(hit != h.end());
+ UNIT_ASSERT_VALUES_EQUAL(*(i64*)rh.GetPayload(it), hit->second);
+ h.erase(key);
+ }
+
+ UNIT_ASSERT(h.empty());
+ }
+
+ Y_UNIT_TEST(FixedMap) {
+ TRobinHoodHashFixedMap<i32, i64> rh;
+ std::unordered_map<i32, i64> h;
+ for (ui64 i = 0; i < 10000; ++i) {
+ auto k = i % 1000;
+ auto [it, inserted] = h.emplace(k, 0);
+ bool isNew;
+ auto iter = rh.Insert(k, isNew);
+ UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), k);
+ UNIT_ASSERT_VALUES_EQUAL(isNew, inserted);
+ it->second += i;
+ if (isNew) {
+ *(i64*)rh.GetMutablePayload(iter) = i;
+ rh.CheckGrow();
+ } else {
+ *(i64*)rh.GetMutablePayload(iter) += i;
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize());
+ }
+
+ for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) {
+ if (!rh.IsValid(it)) {
+ continue;
+ }
+
+ auto key = rh.GetKey(it);
+ auto hit = h.find(key);
+ UNIT_ASSERT(hit != h.end());
+ UNIT_ASSERT_VALUES_EQUAL(*(i64*)rh.GetPayload(it), hit->second);
+ h.erase(key);
+ }
+
+ UNIT_ASSERT(h.empty());
+ }
+
+
+ Y_UNIT_TEST(Set) {
+ TRobinHoodHashSet<i32> rh;
+ std::unordered_set<i32> h;
+ for (ui64 i = 0; i < 10000; ++i) {
+ auto k = i % 1000;
+ auto[it, inserted] = h.emplace(k);
+ bool isNew;
+ auto iter = rh.Insert(k, isNew);
+ UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), k);
+ UNIT_ASSERT_VALUES_EQUAL(isNew, inserted);
+ if (isNew) {
+ rh.CheckGrow();
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize());
+ }
+
+ for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) {
+ if (!rh.IsValid(it)) {
+ continue;
+ }
+
+ auto key = rh.GetKey(it);
+ auto hit = h.find(key);
+ UNIT_ASSERT(hit != h.end());
+ h.erase(key);
+ }
+
+ UNIT_ASSERT(h.empty());
+ }
+
+ Y_UNIT_TEST(MapBatch) {
+ using THashTable = TRobinHoodHashMap<i32>;
+ THashTable rh(sizeof(i64));
+ std::unordered_map<i32, i64> h;
+ std::array<TRobinHoodBatchRequestItem<i32>, PrefetchBatchSize> batch;
+ std::array<bool, PrefetchBatchSize> batchInserted;
+ std::array<ui64, PrefetchBatchSize> batchI;
+ ui32 batchLen = 0;
+
+ auto processBatch = [&]() {
+ rh.BatchInsert({batch.data(), batchLen}, [&](size_t i, THashTable::iterator iter, bool isNew) {
+ UNIT_ASSERT_VALUES_EQUAL(isNew, batchInserted[i]);
+ UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), batch[i].GetKey());
+ if (isNew) {
+ *(i64*)rh.GetMutablePayload(iter) = batchI[i];
+ } else {
+ *(i64*)rh.GetMutablePayload(iter) += batchI[i];
+ }
+ });
+
+ UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize());
+ };
+
+ for (ui64 i = 0; i < 10000; ++i) {
+ if (batchLen == batch.size()) {
+ processBatch();
+ batchLen = 0;
+ }
+
+ auto k = i % 1000;
+ auto [it, inserted] = h.emplace(k, 0);
+ batchI[batchLen] = i;
+ batchInserted[batchLen] = inserted;
+ batch[batchLen].ConstructKey(k);
+ ++batchLen;
+ it->second += i;
+ }
+
+ processBatch();
+ for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) {
+ if (!rh.IsValid(it)) {
+ continue;
+ }
+
+ auto key = rh.GetKey(it);
+ auto hit = h.find(key);
+ UNIT_ASSERT(hit != h.end());
+ UNIT_ASSERT_VALUES_EQUAL(*(i64*)rh.GetPayload(it), hit->second);
+ h.erase(key);
+ }
+
+ UNIT_ASSERT(h.empty());
+ }
+
+ Y_UNIT_TEST(FixedMapBatch) {
+ using THashTable = TRobinHoodHashFixedMap<i32, i64>;
+ THashTable rh(sizeof(i64));
+ std::unordered_map<i32, i64> h;
+ std::array<TRobinHoodBatchRequestItem<i32>, PrefetchBatchSize> batch;
+ std::array<bool, PrefetchBatchSize> batchInserted;
+ std::array<ui64, PrefetchBatchSize> batchI;
+ ui32 batchLen = 0;
+
+ auto processBatch = [&]() {
+ rh.BatchInsert({batch.data(), batchLen}, [&](size_t i, THashTable::iterator iter, bool isNew) {
+ UNIT_ASSERT_VALUES_EQUAL(isNew, batchInserted[i]);
+ UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), batch[i].GetKey());
+ if (isNew) {
+ *(i64*)rh.GetMutablePayload(iter) = batchI[i];
+ } else {
+ *(i64*)rh.GetMutablePayload(iter) += batchI[i];
+ }
+ });
+
+ UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize());
+ };
+
+ for (ui64 i = 0; i < 10000; ++i) {
+ if (batchLen == batch.size()) {
+ processBatch();
+ batchLen = 0;
+ }
+
+ auto k = i % 1000;
+ auto [it, inserted] = h.emplace(k, 0);
+ batchI[batchLen] = i;
+ batchInserted[batchLen] = inserted;
+ batch[batchLen].ConstructKey(k);
+ ++batchLen;
+ it->second += i;
+ }
+
+ processBatch();
+ for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) {
+ if (!rh.IsValid(it)) {
+ continue;
+ }
+
+ auto key = rh.GetKey(it);
+ auto hit = h.find(key);
+ UNIT_ASSERT(hit != h.end());
+ UNIT_ASSERT_VALUES_EQUAL(*(i64*)rh.GetPayload(it), hit->second);
+ h.erase(key);
+ }
+
+ UNIT_ASSERT(h.empty());
+ }
+
+ Y_UNIT_TEST(SetBatch) {
+ using THashTable = TRobinHoodHashSet<i32>;
+ THashTable rh(sizeof(i64));
+ std::unordered_set<i32> h;
+ std::array<TRobinHoodBatchRequestItem<i32>, PrefetchBatchSize> batch;
+ std::array<bool, PrefetchBatchSize> batchInserted;
+ ui32 batchLen = 0;
+
+ auto processBatch = [&]() {
+ rh.BatchInsert({batch.data(), batchLen}, [&](size_t i, THashTable::iterator iter, bool isNew) {
+ UNIT_ASSERT_VALUES_EQUAL(isNew, batchInserted[i]);
+ UNIT_ASSERT_VALUES_EQUAL(rh.GetKey(iter), batch[i].GetKey());
+ });
+
+ UNIT_ASSERT_VALUES_EQUAL(h.size(), rh.GetSize());
+ };
+
+ for (ui64 i = 0; i < 10000; ++i) {
+ if (batchLen == batch.size()) {
+ processBatch();
+ batchLen = 0;
+ }
+
+ auto k = i % 1000;
+ auto [it, inserted] = h.emplace(k);
+ batchInserted[batchLen] = inserted;
+ batch[batchLen].ConstructKey(k);
+ ++batchLen;
+ }
+
+ processBatch();
+ for (auto it = rh.Begin(); it != rh.End(); rh.Advance(it)) {
+ if (!rh.IsValid(it)) {
+ continue;
+ }
+
+ auto key = rh.GetKey(it);
+ auto hit = h.find(key);
+ UNIT_ASSERT(hit != h.end());
+ h.erase(key);
+ }
+
+ UNIT_ASSERT(h.empty());
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp
new file mode 100644
index 00000000000..e3545ed1fc2
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_safe_circular_buffer_ut.cpp
@@ -0,0 +1,143 @@
+#include "../mkql_safe_circular_buffer.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <ydb/library/yql/public/udf/udf_value.h>
+
+namespace NKikimr {
+using namespace NUdf;
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLSafeCircularBuffer) {
+ typedef TSafeCircularBuffer<TUnboxedValue> TBufUnboxed;
+
+ Y_UNIT_TEST(TestUnboxedNoFailOnEmpty) {
+ TBufUnboxed bufferOptional(1, TUnboxedValuePod());
+ TBufUnboxed buffer(1, TUnboxedValue::Void());
+ UNIT_ASSERT(buffer.Get(0));
+ UNIT_ASSERT(buffer.Get(1));
+ UNIT_ASSERT(buffer.Get(3));
+ UNIT_ASSERT(buffer.Get(-1));
+ for (auto i = 0; i < 5; ++i) {
+ buffer.PopFront();
+ }
+ }
+
+ Y_UNIT_TEST(TestUnboxedNormalUsage) {
+ TBufUnboxed buffer(5, TUnboxedValuePod());
+ buffer.PushBack(TUnboxedValue::Embedded("It"));
+ UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "It");
+ buffer.PushBack(TUnboxedValue::Embedded("is"));
+ UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "It");
+ buffer.PushBack(TUnboxedValue::Embedded("funny"));
+ UNIT_ASSERT_EQUAL(buffer.UsedSize(), 3);
+ UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "It");
+ UNIT_ASSERT_EQUAL(buffer.Get(2).AsStringRef(), "funny");
+ UNIT_ASSERT(!buffer.Get(3));
+ buffer.PopFront();
+ UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "is");
+ UNIT_ASSERT_EQUAL(buffer.Get(1).AsStringRef(), "funny");
+ buffer.PushBack(TUnboxedValue::Embedded("bunny"));
+ UNIT_ASSERT_EQUAL(buffer.Get(1).AsStringRef(), "funny");
+ UNIT_ASSERT_EQUAL(buffer.Get(2).AsStringRef(), "bunny");
+ UNIT_ASSERT(!buffer.Get(3));
+ buffer.PopFront();
+ UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "funny");
+ UNIT_ASSERT_EQUAL(buffer.Get(1).AsStringRef(), "bunny");
+ UNIT_ASSERT_EQUAL(buffer.UsedSize(), 2);
+ buffer.PopFront();
+ UNIT_ASSERT_EQUAL(buffer.Get(0).AsStringRef(), "bunny");
+ UNIT_ASSERT_EQUAL(buffer.UsedSize(), 1);
+ for (auto i = 0; i < 3; ++i) {
+ buffer.PopFront();
+ UNIT_ASSERT_EQUAL(buffer.UsedSize(), 0);
+ UNIT_ASSERT(!buffer.Get(0));
+ }
+ }
+
+ Y_UNIT_TEST(TestOverflowNoInitSize) {
+ TBufUnboxed buffer(3, TUnboxedValuePod(), 0);
+ buffer.PushBack(TUnboxedValue::Embedded("1"));
+ buffer.PushBack(TUnboxedValue::Embedded("2"));
+ buffer.PushBack(TUnboxedValue::Embedded("3"));
+ UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("4")), yexception);
+ }
+
+ Y_UNIT_TEST(TestOverflowWithInitSize) {
+ TBufUnboxed buffer(3, TUnboxedValuePod(), 3);
+ buffer.PopFront();
+ buffer.PushBack(TUnboxedValue::Embedded("1"));
+ buffer.PopFront();
+ buffer.PushBack(TUnboxedValue::Embedded("2"));
+ UNIT_ASSERT_EQUAL(buffer.UsedSize(), 3);
+ UNIT_ASSERT_EQUAL(buffer.Get(1).AsStringRef(), "1");
+ UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("3")), yexception);
+ }
+
+ Y_UNIT_TEST(TestOverflowOnEmpty) {
+ TBufUnboxed buffer(0, TUnboxedValuePod());
+ buffer.PopFront();
+ UNIT_ASSERT_EXCEPTION(buffer.PushBack(TUnboxedValue::Embedded("1")), yexception);
+ }
+
+ Y_UNIT_TEST(TestUnbounded) {
+ TBufUnboxed buffer({}, TUnboxedValuePod(), 3);
+ for (size_t i = 0; i < 100; ++i) {
+ buffer.PushBack(TUnboxedValue::Embedded(ToString(i)));
+ }
+
+ UNIT_ASSERT(!buffer.Get(0));
+ UNIT_ASSERT(!buffer.Get(1));
+ UNIT_ASSERT(!buffer.Get(2));
+
+ for (size_t i = 0; i < 100; ++i) {
+ UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(i + 3).AsStringRef()), ToString(i));
+ }
+
+ for (size_t i = 0; i < 100; ++i) {
+ buffer.PopFront();
+ }
+
+ UNIT_ASSERT_EQUAL(buffer.UsedSize(), 3);
+ UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(0).AsStringRef()), ToString(97));
+ UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(1).AsStringRef()), ToString(98));
+ UNIT_ASSERT_EQUAL(TStringBuf(buffer.Get(2).AsStringRef()), ToString(99));
+
+ buffer.PopFront();
+ buffer.PopFront();
+ buffer.PopFront();
+ UNIT_ASSERT_EQUAL(buffer.UsedSize(), 0);
+ UNIT_ASSERT_EQUAL(buffer.Size(), 103);
+ }
+
+ Y_UNIT_TEST(TestUnboxedFewCycles) {
+ const auto multFillLevel = 0.6;
+ const auto multPopLevel = 0.5;
+ const auto initSize = 10;
+ const auto iterationCount = 4;
+ TBufUnboxed buffer(initSize, NUdf::TUnboxedValuePod());
+ unsigned lastDirectIndex = 0;
+ unsigned lastChecked = 0;
+ for (unsigned iteration = 0; iteration < iterationCount; ++iteration) {
+ for (auto i = 0; i < multFillLevel * initSize; ++i) {
+ buffer.PushBack(NUdf::TUnboxedValuePod(++lastDirectIndex));
+ }
+ UNIT_ASSERT(buffer.UsedSize() > 0);
+ UNIT_ASSERT_EQUAL(buffer.Get(buffer.UsedSize() - 1).Get<unsigned>(), lastDirectIndex);
+ for (auto i = 0; i < multPopLevel * initSize; ++i) {
+ auto curUnboxed = buffer.Get(0);
+ auto cur = curUnboxed.Get<unsigned>();
+ UNIT_ASSERT(lastChecked < cur);
+ lastChecked = cur;
+ buffer.PopFront();
+ }
+ UNIT_ASSERT_EQUAL(buffer.UsedSize(), iteration + 1);
+ }
+ UNIT_ASSERT_EQUAL(buffer.UsedSize(), lastDirectIndex - lastChecked);
+ UNIT_ASSERT_EQUAL(buffer.Get(0).Get<unsigned>(), lastChecked + 1);
+ UNIT_ASSERT_EQUAL(buffer.Get(buffer.UsedSize() - 1).Get<unsigned>(), lastDirectIndex);
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp
new file mode 100644
index 00000000000..49e98020e63
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_sort_ut.cpp
@@ -0,0 +1,523 @@
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+#include <ydb/library/yql/utils/sort.h>
+
+#include "mkql_computation_node_ut.h"
+
+#include <random>
+#include <ctime>
+#include <algorithm>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+template <bool LLVM>
+TRuntimeNode MakeStream(TSetup<LLVM>& setup) {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ TCallableBuilder callableBuilder(*setup.Env, "TestYieldStream",
+ pgmBuilder.NewStreamType(
+ pgmBuilder.NewStructType({
+ {TStringBuf("a"), pgmBuilder.NewDataType(NUdf::EDataSlot::Uint64)},
+ {TStringBuf("b"), pgmBuilder.NewDataType(NUdf::EDataSlot::String)}
+ })
+ )
+ );
+
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
+TRuntimeNode StreamToString(TProgramBuilder& pgmBuilder, TRuntimeNode stream) {
+ return pgmBuilder.Condense(stream,
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|"),
+ [&] (TRuntimeNode, TRuntimeNode) { return pgmBuilder.NewDataLiteral<bool>(false); },
+ [&] (TRuntimeNode item, TRuntimeNode state) {
+ auto str = pgmBuilder.Concat(
+ pgmBuilder.Concat(
+ pgmBuilder.ToString(pgmBuilder.Member(item, "a")),
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("-")
+ ),
+ pgmBuilder.Member(item, "b")
+ );
+
+ return pgmBuilder.Concat(pgmBuilder.Concat(state, str),
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|"));
+ }
+ );
+}
+
+} // namespace
+
+Y_UNIT_TEST_SUITE(TMiniKQLSortTest) {
+ Y_UNIT_TEST_LLVM(TestStreamSort) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ std::vector<TRuntimeNode> order {
+ pgmBuilder.NewDataLiteral<bool>(true),
+ pgmBuilder.NewDataLiteral<bool>(false)
+ };
+ auto sort = pgmBuilder.Sort(stream, pgmBuilder.NewTuple(order),
+ [&pgmBuilder](TRuntimeNode item) {
+ std::vector<TRuntimeNode> keys {
+ pgmBuilder.Member(item, "a"),
+ pgmBuilder.Member(item, "b")
+ };
+
+ return pgmBuilder.NewTuple(keys);
+ });
+
+ auto pgmResult = StreamToString(pgmBuilder, sort);
+
+ auto graph = setup.BuildGraph(pgmResult);
+ auto value = graph->GetValue();
+
+
+ NUdf::TUnboxedValue result;
+ auto yieldCount = 0U;
+ auto status = NUdf::EFetchStatus::Ok;
+ while (status != NUdf::EFetchStatus::Finish) {
+ status = value.Fetch(result);
+ if (status == NUdf::EFetchStatus::Yield) {
+ ++yieldCount;
+ }
+ }
+
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
+ UNIT_ASSERT_EQUAL(yieldCount, 3U);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()),
+ "|0-8|0-4|0-11|0-0|1-9|1-6|1-13|1-1|2-7|2-2|2-14|2-10|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestFlowSortByLambdaComparator) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ const auto stream = MakeStream(setup);
+
+ const auto sort =
+ pgmBuilder.FromFlow(pgmBuilder.FlatMap(
+ pgmBuilder.Condense1(pgmBuilder.ToFlow(stream),
+ [&](TRuntimeNode item) { return pgmBuilder.AsList(item); },
+ [&](TRuntimeNode, TRuntimeNode) { return pgmBuilder.NewDataLiteral<bool>(false); },
+ [&](TRuntimeNode item, TRuntimeNode state) { return pgmBuilder.Append(state, item); }
+ ),
+ [&](TRuntimeNode list) {
+ return pgmBuilder.StableSort(list,
+ [&](TRuntimeNode left, TRuntimeNode right) {
+ return pgmBuilder.Or({
+ pgmBuilder.AggrLess(pgmBuilder.Member(left, "a"), pgmBuilder.Member(right, "a")),
+ pgmBuilder.And({
+ pgmBuilder.AggrEquals(pgmBuilder.Member(left, "a"), pgmBuilder.Member(right, "a")),
+ pgmBuilder.AggrGreater(pgmBuilder.Member(left, "b"), pgmBuilder.Member(right, "b"))
+ })
+ });
+ }
+ );
+ }
+ ));
+
+ const auto pgmResult = StreamToString(pgmBuilder, sort);
+
+ const auto graph = setup.BuildGraph(pgmResult);
+ const auto value = graph->GetValue();
+
+
+ NUdf::TUnboxedValue result;
+ auto yieldCount = 0U;
+ auto status = NUdf::EFetchStatus::Ok;
+ while (status != NUdf::EFetchStatus::Finish) {
+ status = value.Fetch(result);
+ if (status == NUdf::EFetchStatus::Yield) {
+ ++yieldCount;
+ }
+ }
+
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
+ UNIT_ASSERT_EQUAL(yieldCount, 3U);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()),
+ "|0-8|0-4|0-11|0-0|1-9|1-6|1-13|1-1|2-7|2-2|2-14|2-10|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestListSort) {
+ const std::array<double, 10U> xxx = {{9E9, -HUGE_VAL, 0.003, HUGE_VAL, +3.1415, -0.003, -7898.8, -3.1415, 3673.0, 0.003}};
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ std::array<TRuntimeNode, 10U> data;
+ std::transform(xxx.cbegin(), xxx.cend(), data.begin(),
+ std::bind(&TProgramBuilder::NewDataLiteral<double>, std::ref(pgmBuilder), std::placeholders::_1 ));
+
+ const auto type = pgmBuilder.NewDataType(NUdf::TDataType<double>::Id);
+ const auto list = pgmBuilder.NewList(type, data);
+
+ const auto pgmReturn = pgmBuilder.Sort(list, pgmBuilder.NewDataLiteral<bool>(false),
+ std::bind(&TProgramBuilder::Abs, std::ref(pgmBuilder), std::placeholders::_1 ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& result = graph->GetValue();
+
+ UNIT_ASSERT_VALUES_EQUAL(result.GetListLength(), xxx.size());
+
+ auto copy = xxx;
+ std::stable_sort(copy.begin(), copy.end(), [](double l, double r){ return std::abs(l) > std::abs(r); });
+
+ for (auto i = 0U; i < copy.size(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(copy[i], result.GetElement(i).template Get<double>());
+ }
+ }
+
+ Y_UNIT_TEST_LLVM(TestListTop) {
+ TSetup<LLVM> setup;
+
+ std::default_random_engine eng;
+ eng.seed(std::time(nullptr));
+
+ std::uniform_real_distribution<double> unifd(-999.0, +999.0);
+ std::uniform_real_distribution<ui64> unifi;
+
+ constexpr ui64 total = 999ULL;
+
+ std::array<TRuntimeNode, total> data;
+ std::vector<std::pair<double, ui64>> test;
+ test.reserve(total);
+
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ std::generate_n(std::back_inserter(test), total, [&]() { return std::make_pair(unifd(eng), unifi(eng) % 100U);});
+
+ std::transform(test.cbegin(), test.cend(), data.begin(), [&](const auto& pair) {
+ return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<double>(pair.first), pgmBuilder.NewDataLiteral<ui64>(pair.second)});
+ });
+
+ const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<double>::Id), pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)});
+ const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(false), pgmBuilder.NewDataLiteral<bool>(true)});
+ const auto list = pgmBuilder.NewList(tupleType, data);
+ const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 1U), pgmBuilder.Abs(pgmBuilder.Nth(item, 0U))}); };
+ const auto n = 17ULL;
+ const auto limit = pgmBuilder.NewDataLiteral<ui64>(n);
+
+ const auto pgmReturn = pgmBuilder.Top(list, limit, order, extractor);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& value = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), n);
+
+ const auto& iter = value.GetListIterator();
+
+ auto copy = test;
+
+ const auto comp = [](const auto& l, const auto& r){ return l.second > r.second || (l.second == r.second && std::abs(l.first) < std::abs(r.first)); };
+ NYql::FastNthElement(copy.begin(), copy.begin() + n - 1ULL, copy.end(), comp);
+ copy.resize(n);
+
+ std::vector<std::pair<double, ui64>> res;
+ res.reserve(n);
+
+ for (NUdf::TUnboxedValue item; iter.Next(item);) {
+ res.emplace_back(item.GetElement(0U).template Get<double>(), item.GetElement(1U).template Get<ui64>());
+ }
+
+ std::sort(copy.begin(), copy.end());
+ std::sort(res.begin(), res.end());
+ UNIT_ASSERT(copy == res);
+ }
+
+ Y_UNIT_TEST_LLVM(TestStreamZeroTop) {
+ TSetup<LLVM> setup;
+
+ constexpr ui64 total = 9ULL;
+
+ std::array<TRuntimeNode, total> data;
+
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ std::generate_n(data.begin(), total, [&]() {
+ return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<ui64>(42ULL)});
+ });
+
+ const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)});
+ const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(false)});
+ const auto list = pgmBuilder.NewList(tupleType, data);
+ const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 0U)}); };
+ const auto limit = pgmBuilder.NewDataLiteral<ui64>(0ULL);
+
+ const auto pgmReturn = pgmBuilder.Top(pgmBuilder.Iterator(list, {}), limit, order, extractor);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& value = graph->GetValue();
+ NUdf::TUnboxedValue stub;
+ UNIT_ASSERT_VALUES_EQUAL(value.Fetch(stub), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestListTopSort) {
+ TSetup<LLVM> setup;
+
+ std::default_random_engine eng;
+ eng.seed(std::time(nullptr));
+
+ std::uniform_real_distribution<double> unifd(-999.0, +999.0);
+ std::uniform_real_distribution<ui64> unifi;
+
+ constexpr ui64 total = 999ULL;
+
+ std::array<TRuntimeNode, total> data;
+ std::vector<std::pair<double, ui64>> test;
+ test.reserve(total);
+
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ std::generate_n(std::back_inserter(test), total, [&]() { return std::make_pair(unifd(eng), unifi(eng) % 100U);});
+
+ std::transform(test.cbegin(), test.cend(), data.begin(), [&](const auto& pair) {
+ return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<double>(pair.first), pgmBuilder.NewDataLiteral<ui64>(pair.second)});
+ });
+
+ const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<double>::Id), pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)});
+ const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(false), pgmBuilder.NewDataLiteral<bool>(true)});
+ const auto list = pgmBuilder.NewList(tupleType, data);
+ const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 1U), pgmBuilder.Abs(pgmBuilder.Nth(item, 0U))}); };
+ const auto n = 17ULL;
+ const auto limit = pgmBuilder.NewDataLiteral<ui64>(n);
+
+ const auto pgmReturn = pgmBuilder.TopSort(list, limit, order, extractor);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& value = graph->GetValue();
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), n);
+
+ const auto& iter = value.GetListIterator();
+
+ auto copy = test;
+
+ const auto comp = [](const auto& l, const auto& r){ return l.second > r.second || (l.second == r.second && std::abs(l.first) < std::abs(r.first)); };
+ NYql::FastPartialSort(copy.begin(), copy.begin() + n, copy.end(), comp);
+ copy.resize(n);
+
+ std::vector<std::pair<double, ui64>> res;
+ res.reserve(n);
+
+ for (NUdf::TUnboxedValue item; iter.Next(item);) {
+ res.emplace_back(item.GetElement(0U).template Get<double>(), item.GetElement(1U).template Get<ui64>());
+ }
+
+ UNIT_ASSERT(copy == res);
+ }
+
+ Y_UNIT_TEST_LLVM(TestStreamTop) {
+ TSetup<LLVM> setup;
+
+ std::default_random_engine eng;
+ eng.seed(std::time(nullptr));
+
+ std::uniform_real_distribution<double> unifd(-999.0, +999.0);
+ std::uniform_real_distribution<ui64> unifi;
+
+ constexpr ui64 total = 999ULL;
+
+ std::array<TRuntimeNode, total> data;
+ std::vector<std::pair<double, ui64>> test;
+ test.reserve(total);
+
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ std::generate_n(std::back_inserter(test), total, [&]() { return std::make_pair(unifd(eng), unifi(eng) % 100U);});
+
+ std::transform(test.cbegin(), test.cend(), data.begin(), [&](const auto& pair) {
+ return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<double>(pair.first), pgmBuilder.NewDataLiteral<ui64>(pair.second)});
+ });
+
+ const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<double>::Id), pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)});
+ const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(true), pgmBuilder.NewDataLiteral<bool>(false)});
+ const auto list = pgmBuilder.NewList(tupleType, data);
+ const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 1U), pgmBuilder.Abs(pgmBuilder.Nth(item, 0U))}); };
+ const auto n = 17ULL;
+ const auto limit = pgmBuilder.NewDataLiteral<ui64>(n);
+
+ const auto pgmReturn = pgmBuilder.Top(pgmBuilder.Iterator(list, {}), limit, order, extractor);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& value = graph->GetValue();
+
+ auto copy = test;
+
+ const auto comp = [](const auto& l, const auto& r){ return l.second < r.second || (l.second == r.second && std::abs(l.first) > std::abs(r.first)); };
+ NYql::FastNthElement(copy.begin(), copy.begin() + n - 1ULL, copy.end(), comp);
+ copy.resize(n);
+
+ std::vector<std::pair<double, ui64>> res;
+ res.reserve(n);
+
+ for (NUdf::TUnboxedValue item; NUdf::EFetchStatus::Ok == value.Fetch(item);) {
+ res.emplace_back(item.GetElement(0U).template Get<double>(), item.GetElement(1U).template Get<ui64>());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(res.size(), n);
+
+ std::sort(copy.begin(), copy.end());
+ std::sort(res.begin(), res.end());
+ UNIT_ASSERT(copy == res);
+ }
+
+ Y_UNIT_TEST_LLVM(TestStreamTopSort) {
+ TSetup<LLVM> setup;
+
+ std::default_random_engine eng;
+ eng.seed(std::time(nullptr));
+
+ std::uniform_real_distribution<double> unifd(-999.0, +999.0);
+ std::uniform_real_distribution<ui64> unifi;
+
+ constexpr ui64 total = 999ULL;
+
+ std::array<TRuntimeNode, total> data;
+ std::vector<std::pair<double, ui64>> test;
+ test.reserve(total);
+
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ std::generate_n(std::back_inserter(test), total, [&]() { return std::make_pair(unifd(eng), unifi(eng) % 100U);});
+
+ std::transform(test.cbegin(), test.cend(), data.begin(), [&](const auto& pair) {
+ return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<double>(pair.first), pgmBuilder.NewDataLiteral<ui64>(pair.second)});
+ });
+
+ const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<double>::Id), pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)});
+ const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(true), pgmBuilder.NewDataLiteral<bool>(false)});
+ const auto list = pgmBuilder.NewList(tupleType, data);
+ const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 1U), pgmBuilder.Abs(pgmBuilder.Nth(item, 0U))}); };
+ const auto n = 17ULL;
+ const auto limit = pgmBuilder.NewDataLiteral<ui64>(n);
+
+ const auto pgmReturn = pgmBuilder.TopSort(pgmBuilder.Iterator(list, {}), limit, order, extractor);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& value = graph->GetValue();
+
+ auto copy = test;
+
+ const auto comp = [](const auto& l, const auto& r){ return l.second < r.second || (l.second == r.second && std::abs(l.first) > std::abs(r.first)); };
+ NYql::FastPartialSort(copy.begin(), copy.begin() + n, copy.end(), comp);
+ copy.resize(n);
+
+ std::vector<std::pair<double, ui64>> res;
+ res.reserve(n);
+
+ for (NUdf::TUnboxedValue item; NUdf::EFetchStatus::Ok == value.Fetch(item);) {
+ res.emplace_back(item.GetElement(0U).template Get<double>(), item.GetElement(1U).template Get<ui64>());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(res.size(), n);
+ UNIT_ASSERT(copy == res);
+ }
+
+ Y_UNIT_TEST_LLVM(TestStreamTopSortBySingleField) {
+ TSetup<LLVM> setup;
+
+ std::default_random_engine eng;
+ eng.seed(std::time(nullptr));
+
+ std::uniform_real_distribution<ui64> unifi;
+
+ constexpr ui64 total = 999ULL;
+
+ std::array<TRuntimeNode, total> data;
+ std::vector<ui64> test;
+ test.reserve(total);
+
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ std::generate_n(std::back_inserter(test), total, [&]() { return unifi(eng) % 100ULL; });
+
+ std::transform(test.cbegin(), test.cend(), data.begin(), [&](const ui64& v) {
+ return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<ui64>(v)});
+ });
+
+ const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)});
+ const auto order = pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<bool>(false)});
+ const auto list = pgmBuilder.NewList(tupleType, data);
+ const auto extractor = [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.NewTuple({pgmBuilder.Nth(item, 0U)}); };
+ const auto n = 17ULL;
+ const auto limit = pgmBuilder.NewDataLiteral<ui64>(n);
+
+ const auto pgmReturn = pgmBuilder.TopSort(pgmBuilder.Iterator(list, {}), limit, order, extractor);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& value = graph->GetValue();
+
+ auto copy = test;
+
+ NYql::FastPartialSort(copy.begin(), copy.begin() + n, copy.end(), std::greater<ui64>());
+ copy.resize(n);
+
+ std::vector<ui64> res;
+ res.reserve(n);
+
+ for (NUdf::TUnboxedValue item; NUdf::EFetchStatus::Ok == value.Fetch(item);) {
+ res.emplace_back(item.GetElement(0U).template Get<ui64>());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(res.size(), n);
+ UNIT_ASSERT(copy == res);
+ }
+
+ Y_UNIT_TEST_LLVM(TestFlowTopSortWithoutKey) {
+ TSetup<LLVM> setup;
+
+ std::default_random_engine eng;
+ eng.seed(std::time(nullptr));
+
+ std::uniform_real_distribution<ui64> unifi;
+
+ constexpr ui64 total = 99ULL;
+
+ std::array<TRuntimeNode, total> data;
+ std::vector<ui64> test;
+ test.reserve(total);
+
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ std::generate_n(std::back_inserter(test), total, [&]() { return unifi(eng) % 100ULL; });
+
+ std::transform(test.cbegin(), test.cend(), data.begin(), [&](const ui64& v) {
+ return pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<ui64>(v)});
+ });
+
+ const auto tupleType = pgmBuilder.NewTupleType({pgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)});
+ const auto order = pgmBuilder.NewTuple({});
+ const auto list = pgmBuilder.NewList(tupleType, data);
+ const auto extractor = [&pgmBuilder](TRuntimeNode) { return pgmBuilder.NewTuple({}); };
+ const auto n = 17ULL;
+ const auto limit = pgmBuilder.NewDataLiteral<ui64>(n);
+
+ const auto pgmReturn = pgmBuilder.FromFlow(pgmBuilder.TopSort(pgmBuilder.ToFlow(list), limit, order, extractor));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto& value = graph->GetValue();
+
+ auto copy = test;
+ copy.resize(n);
+
+ std::vector<ui64> res;
+ res.reserve(n);
+
+ for (NUdf::TUnboxedValue item; NUdf::EFetchStatus::Ok == value.Fetch(item);) {
+ res.emplace_back(item.GetElement(0U).template Get<ui64>());
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(res.size(), n);
+ UNIT_ASSERT(copy == res);
+ }
+}
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp
new file mode 100644
index 00000000000..37472db9d60
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_switch_ut.cpp
@@ -0,0 +1,276 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLSwitchTest) {
+ Y_UNIT_TEST_LLVM(TestStreamOfVariantsSwap) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1U);
+ const auto data2 = pb.NewDataLiteral<ui32>(2U);
+ const auto data3 = pb.NewDataLiteral<ui32>(3U);
+
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("123");
+ const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("456");
+ const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("789");
+
+ const auto intType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+
+ const auto varInType = pb.NewVariantType(pb.NewTupleType({intType, strType}));
+
+ const auto var1 = pb.NewVariant(data1, 0U, varInType);
+ const auto var2 = pb.NewVariant(data2, 0U, varInType);
+ const auto var3 = pb.NewVariant(data3, 0U, varInType);
+ const auto var4 = pb.NewVariant(data4, 1U, varInType);
+ const auto var5 = pb.NewVariant(data5, 1U, varInType);
+ const auto var6 = pb.NewVariant(data6, 1U, varInType);
+
+ const auto varOutType = pb.NewVariantType(pb.NewTupleType({strType, intType}));
+
+ const auto list = pb.NewList(varInType, {var1, var2, var3, var4, var5, var6});
+
+ const auto pgmReturn = pb.Switch(pb.Iterator(list, {}),
+ {{{0U}, pb.NewStreamType(intType), std::nullopt}, {{1U}, pb.NewStreamType(strType), std::nullopt}},
+ [&](ui32 index, TRuntimeNode stream) {
+ switch (index) {
+ case 0U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.ToString(item), 0U, varOutType); });
+ case 1U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.StrictFromString(item, intType), 1U, varOutType); });
+ }
+ Y_FAIL("Wrong case!");
+ },
+ 0ULL,
+ pb.NewStreamType(varOutType)
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U);
+ UNBOXED_VALUE_STR_EQUAL(item, "1");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U);
+ UNBOXED_VALUE_STR_EQUAL(item, "2");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U);
+ UNBOXED_VALUE_STR_EQUAL(item, "3");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 123U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 456U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 789U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestStreamOfVariantsTwoInOne) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1U);
+ const auto data2 = pb.NewDataLiteral<ui32>(2U);
+ const auto data3 = pb.NewDataLiteral<ui32>(3U);
+
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("123");
+ const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("456");
+ const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("789");
+
+ const auto intType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+
+ const auto varInType = pb.NewVariantType(pb.NewTupleType({intType, strType}));
+
+ const auto var1 = pb.NewVariant(data1, 0U, varInType);
+ const auto var2 = pb.NewVariant(data2, 0U, varInType);
+ const auto var3 = pb.NewVariant(data3, 0U, varInType);
+ const auto var4 = pb.NewVariant(data4, 1U, varInType);
+ const auto var5 = pb.NewVariant(data5, 1U, varInType);
+ const auto var6 = pb.NewVariant(data6, 1U, varInType);
+
+ const auto varOutType = pb.NewVariantType(pb.NewTupleType({strType, intType}));
+
+ const auto list = pb.NewList(varInType, {var1, var2, var3, var4, var5, var6});
+
+ const auto pgmReturn = pb.Switch(pb.Iterator(list, {}),
+ {{{0U}, pb.NewStreamType(intType), 1U}, {{1U}, pb.NewStreamType(strType), std::nullopt}},
+ [&](ui32 index, TRuntimeNode stream) {
+ switch (index) {
+ case 0U: return pb.Map(stream, [&](TRuntimeNode item) { return item; });
+ case 1U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.StrictFromString(item, intType), 1U, varOutType); });
+ }
+ Y_FAIL("Wrong case!");
+ },
+ 0ULL,
+ pb.NewStreamType(varOutType)
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 2U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 3U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 123U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 456U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 789U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFlowOfVariantsSwap) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1U);
+ const auto data2 = pb.NewDataLiteral<ui32>(2U);
+ const auto data3 = pb.NewDataLiteral<ui32>(3U);
+
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("123");
+ const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("456");
+ const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("789");
+
+ const auto intType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+
+ const auto varInType = pb.NewVariantType(pb.NewTupleType({intType, strType}));
+
+ const auto var1 = pb.NewVariant(data1, 0U, varInType);
+ const auto var2 = pb.NewVariant(data2, 0U, varInType);
+ const auto var3 = pb.NewVariant(data3, 0U, varInType);
+ const auto var4 = pb.NewVariant(data4, 1U, varInType);
+ const auto var5 = pb.NewVariant(data5, 1U, varInType);
+ const auto var6 = pb.NewVariant(data6, 1U, varInType);
+
+ const auto varOutType = pb.NewVariantType(pb.NewTupleType({strType, intType}));
+
+ const auto list = pb.NewList(varInType, {var1, var2, var3, var4, var5, var6});
+
+ const auto pgmReturn = pb.FromFlow(pb.Switch(pb.ToFlow(list),
+ {{{0U}, pb.NewFlowType(intType), std::nullopt}, {{1U}, pb.NewFlowType(strType), std::nullopt}},
+ [&](ui32 index, TRuntimeNode stream) {
+ switch (index) {
+ case 0U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.ToString(item), 0U, varOutType); });
+ case 1U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.StrictFromString(item, intType), 1U, varOutType); });
+ }
+ Y_FAIL("Wrong case!");
+ },
+ 0ULL,
+ pb.NewFlowType(varOutType)
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U);
+ UNBOXED_VALUE_STR_EQUAL(item, "1");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U);
+ UNBOXED_VALUE_STR_EQUAL(item, "2");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 0U);
+ UNBOXED_VALUE_STR_EQUAL(item, "3");
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 123U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 456U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 789U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFlowOfVariantsTwoInOne) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1U);
+ const auto data2 = pb.NewDataLiteral<ui32>(2U);
+ const auto data3 = pb.NewDataLiteral<ui32>(3U);
+
+ const auto data4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("123");
+ const auto data5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("456");
+ const auto data6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("789");
+
+ const auto intType = pb.NewDataType(NUdf::TDataType<ui32>::Id);
+ const auto strType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+
+ const auto varInType = pb.NewVariantType(pb.NewTupleType({intType, strType}));
+
+ const auto var1 = pb.NewVariant(data1, 0U, varInType);
+ const auto var2 = pb.NewVariant(data2, 0U, varInType);
+ const auto var3 = pb.NewVariant(data3, 0U, varInType);
+ const auto var4 = pb.NewVariant(data4, 1U, varInType);
+ const auto var5 = pb.NewVariant(data5, 1U, varInType);
+ const auto var6 = pb.NewVariant(data6, 1U, varInType);
+
+ const auto varOutType = pb.NewVariantType(pb.NewTupleType({strType, intType}));
+
+ const auto list = pb.NewList(varInType, {var1, var2, var3, var4, var5, var6});
+
+ const auto pgmReturn = pb.FromFlow(pb.Switch(pb.ToFlow(list),
+ {{{0U}, pb.NewFlowType(intType), 1U}, {{1U}, pb.NewFlowType(strType), std::nullopt}},
+ [&](ui32 index, TRuntimeNode stream) {
+ switch (index) {
+ case 0U: return pb.Map(stream, [&](TRuntimeNode item) { return item; });
+ case 1U: return pb.Map(stream, [&](TRuntimeNode item) { return pb.NewVariant(pb.StrictFromString(item, intType), 1U, varOutType); });
+ }
+ Y_FAIL("Wrong case!");
+ },
+ 0ULL,
+ pb.NewFlowType(varOutType)
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 2U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 3U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 123U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 456U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetVariantIndex(), 1U);
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), 789U);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
new file mode 100644
index 00000000000..75bf0fe352a
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
@@ -0,0 +1,549 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+#include <util/random/shuffle.h>
+#include <map>
+#include <optional>
+
+namespace NKikimr::NMiniKQL {
+
+static const TStringBuf data[] = {
+ "13d49d4db08e57d645fe4d44bbed4738f386af6e9e742cf186961063feb9919b",
+ "14d285e88582d87c41d3e6d2e9352686d0363ea74a297fe02f901f18c19978a3",
+ "1795ad46329c4fc6b3355dc22d252c5fe390a971ddf009b54fdeceb93d3b8930",
+ "18042e88fb4cf6b09cb8e6c5588ae525fc7a37bd2248a857d83ac1d1dcdf0a64",
+ "1b30b154ac814f7e4ed7e7488e037d781b78fbc336cac027f4c301ad9368514e",
+ "1a0b94cebdc038bb293575af4f6954e9dbf19801e581ad03be90b4aef36347d7",
+ "1c9ac5b87de7d68efae1bdf1ad47e58d28a6a70e966f19899798c61b2a65b6e2",
+ "1618c1e3d9dbc3edaccb7934eca55d2d96cb59d2633655f57401dba99deec3ef",
+ "1bd7a6ff86a1940283e202b142ecba685fea86f93f2aafad8cd37d80582aca95",
+ "0fba3f2f741b0579a3eec906f5341e0556fbd74088fcdfbe776bd6122fa81681",
+ "19768b3228cef7a82e0f1c367d78c42596fa511c735bd85d7cafca0965045562",
+ "1a9c0a14272795d7ad39a4725754c3f1d013a761c41fba16e06ae247833fd42b",
+ "1562ce72ff7229866723e81e35db30d08c8b8dc7b7d076cff787f157d70763e6",
+ "0faf214bafe219413618fdf186bb9290e6a610755d90947cd686b72899e78445",
+ "14f3fe97da837197b98f6303ac5aa1b6f34bffe9841fe65f084a27f4bd4ced8a",
+ "198c0706af7107ababebf1500875ba64508519b21aa534d0f55e8a32e394799d",
+ "1bb66a4593b77b1650a4a530bae30e454c2815477769d67fe6c6b337ae4acafe",
+ "0f67ef1ca6ef6b2d699dfac0360e8f24dc8960428cff058fe285d63ab55ef6d3",
+ "1097009fe853793802120394fbb6404df82725240d410c69375e5a53ec5395b8",
+ "1b1262275eae8a055732253e8d717c19ebde386b25e51dd546413e7ee997c5e1",
+ "1c4a73588541a0c869b4ee27c32cc4218f3c8db13713c61cedc387336a2720c9",
+ "1c73442f0ac53d8b38f779231680fab806a6cb9c86d25d9db5fa67c0ebf8e803",
+ "19152f0c06baf7962ca287a303b85437f321d725985f1586ac8358bdb6a0df63",
+ "13436f337815f5929559e6f621b850ed60b36f23ce9d8d06db981b70d40ad3db",
+ "298268d866eea5d6fcae470fdbb6d7787d73ab1e50b8126d6452d81264fbdafd",
+ "1a67b4e4c213baa140c5a00352cdbc9256b4e2fe81482c380b08ebe2e6b76e1b",
+ "19824d2008be54e35a0e2a9d2df9746e96f73367518b111695e1c3857966c241",
+ "2997c49ed21482d30b8ef89bd26bfdb6384dda6825032145fe0a3ad9d2f2a7e3",
+ "137ccc1d4ab00210bd9af5ee875cb799bd818f4803470abca68a9655ea73be01",
+ "12d4cf2eb41c90ede84bece72f76e97d7d0144c45341a0176f656b295cb838c3",
+ "11d02da4f449e6aeee4f213409baed6eaab35496688d743991870ba093676c44",
+ "163fb1ef04a1453a44fb897014287f7ceefe0b553d99718f986ada32cec6ca29",
+ "16f579a7eda4d7f5cde29724bf35e1b36e95fbeb72914ba2ba8f19192b92dab7",
+ "0f60c1387bf29d8d81174bd43c136e75f0f33b8b4d2712de0cc3a23f04fac76e",
+ "0f83662d3b4cc9aaa0f76c8801d2d32909c050286d514acc108b6c3d9964679b",
+ "1a30b7c4bf1c4eaaa92802cb90a27b5073d4a5ec095271490df8443b8f5df88f",
+ "105af591b517f126c985f54e833d61907ff49945ab123a606caa6d9bda0e3d66",
+ "1a5196fdfc1b81974905a66e6f1ff31403fc84b4d84effde521e848153f42e10",
+ "17d6cb5ba9489d8397cb1e1d054e45cff6c7225aeeba5c9e76cacd9da6c9a0c1",
+ "127ab4e2169329082bdd546e94c4fb6643999b14a26e08eaa719689789767014",
+ "143883410f000b5f3ff4c6434b0654516e9502d0a50a2b3ecdc20c8d3e198915",
+ "16ccd345646dd3d39e6bd157b51513c1b319bd1f441199003052a74b2eddb53d",
+ "11e9f02dc56d575fac5a306a6e83f597ffda1bd81a01f13fdda059ab24d90892",
+ "13f75a9e662faea5fc93f0f83d363c42083345cdcc42f1b0d320d11409ef3052",
+ "18cca97e8c6ede52e0b7d8c53c85c0fac68f6d1b7c6622a4cebc21433e6d8eea",
+ "160d6b818fab5ad00a1e81e46888c5ff3e5f2c175c013ce17d88c31df4475aba",
+ "1c4d09dff19175af7fc0d8e8fd23e9288fc2839dedfc067dcf9f5a3e3a9d92aa",
+ "16e25b2a6eef4cde6879c20c94c4360604b9099c29e1abaf9fc079fe67cfcaac",
+ "2a577ab7e2541e2cc2cc20e6a76c4ea9b77501808db9c4045be82c680cf227d5",
+ "11b4753fd9cc33656dbd59769b3202b7f68bd067bf7f64bd54676f6f60366ef1",
+ "1932a0aecc4a569d7d3fbcdd329b92c0b4dbd870d6be48ec4f18285ab3183676",
+ "2a2e6b62a4383cb48ffbb69b2f356ceb0410593f5b5500142498692dec7c125f",
+};
+
+
+Y_UNIT_TEST_SUITE(TMiniKQLToDictTest) {
+ Y_UNIT_TEST_LLVM(TestCompactUtf8Set) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TVector<TRuntimeNode> items;
+ for (auto s: data) {
+ items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s));
+ }
+ Shuffle(items.begin(), items.end());
+ auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ auto list = pb.NewList(dataType, items);
+ auto dict = pb.ToHashedDict(list, false, [](TRuntimeNode n) { return n; }, [&pb](TRuntimeNode /*n*/) { return pb.NewVoid(); }, true);
+ auto pgmReturn = pb.Contains(dict, items.front());
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto res = graph->GetValue().template Get<bool>();
+ UNIT_ASSERT_VALUES_EQUAL(res, true);
+ }
+
+ Y_UNIT_TEST_LLVM(TestUtf8Set) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TVector<TRuntimeNode> items;
+ for (auto s: data) {
+ items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s));
+ }
+ Shuffle(items.begin(), items.end());
+ auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ auto list = pb.NewList(dataType, items);
+ auto dict = pb.ToHashedDict(list, false, [](TRuntimeNode n) { return n; }, [&pb](TRuntimeNode /*n*/) { return pb.NewVoid(); }, false);
+ auto pgmReturn = pb.Contains(dict, items.front());
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto res = graph->GetValue().template Get<bool>();
+ UNIT_ASSERT_VALUES_EQUAL(res, true);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSqueezeToDict) {
+ auto test = [](bool stream, bool hashed, bool multi, bool compact, bool withPayload) {
+ Cerr << "TestSqueezeToDict [on: " << (stream ? "stream" : "flow")
+ << "type: " << (hashed ? "hashed" : "sorted") << ", multi: " << multi
+ << ", compact: " << compact << ", payload: " << withPayload << "]" << Endl;
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TVector<TRuntimeNode> items;
+ for (auto s : data) {
+ items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s));
+ }
+ Shuffle(items.begin(), items.end());
+
+ auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ auto list = pb.NewList(dataType, items);
+ auto input = stream ? pb.Iterator(list, items) : pb.ToFlow(list);
+ auto pgmReturn = hashed
+ ? pb.SqueezeToHashedDict(input, multi, [](TRuntimeNode n) { return n; },
+ [&pb, withPayload](TRuntimeNode n) { return withPayload ? n : pb.NewVoid(); }, compact)
+ : pb.SqueezeToSortedDict(input, multi, [](TRuntimeNode n) { return n; },
+ [&pb, withPayload](TRuntimeNode n) { return withPayload ? n : pb.NewVoid(); }, compact);
+ if (!stream) {
+ pgmReturn = pb.FromFlow(pgmReturn);
+ }
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ NUdf::TUnboxedValue res = graph->GetValue();
+ UNIT_ASSERT(!res.IsSpecial());
+
+ NUdf::TUnboxedValue v;
+ auto status = res.Fetch(v);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, status);
+
+ for (auto s : data) {
+ UNIT_ASSERT_C(v.Contains(NUdf::TUnboxedValue(MakeString(s))), s);
+ }
+ UNIT_ASSERT(!v.Contains(NUdf::TUnboxedValue(MakeString("green cucumber"))));
+
+ status = res.Fetch(v);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, status);
+ };
+
+ for (auto stream : {true, false}) {
+ for (auto hashed : {true, false}) {
+ for (auto multi : {true, false}) {
+ for (auto compact : {true, false}) {
+ for (auto withPayload : {true, false}) {
+ test(stream, hashed, multi, compact, withPayload);
+ }
+ }
+ }
+ }
+ }
+ }
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 23u
+ Y_UNIT_TEST_LLVM(TestNarrowSqueezeToDict) {
+ auto test = [](bool hashed, bool multi, bool compact, bool withPayload) {
+ Cerr << "TestNarrowSqueezeToDict [type: " << (hashed ? "hashed" : "sorted") << ", multi: " << multi
+ << ", compact: " << compact << ", payload: " << withPayload << "]" << Endl;
+
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TVector<TRuntimeNode> items;
+ for (auto s : data) {
+ items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s));
+ }
+ Shuffle(items.begin(), items.end());
+
+ auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ auto list = pb.NewList(dataType, items);
+ auto input = pb.ExpandMap(pb.ToFlow(list), [](TRuntimeNode n) ->TRuntimeNode::TList { return {n}; });
+ auto pgmReturn = hashed
+ ? pb.NarrowSqueezeToHashedDict(input, multi, [](TRuntimeNode::TList n) { return n.front(); },
+ [&pb, withPayload](TRuntimeNode::TList n) { return withPayload ? n.back() : pb.NewVoid(); }, compact)
+ : pb.NarrowSqueezeToSortedDict(input, multi, [](TRuntimeNode::TList n) { return n.front(); },
+ [&pb, withPayload](TRuntimeNode::TList n) { return withPayload ? n.back() : pb.NewVoid(); }, compact);
+ pgmReturn = pb.FromFlow(pgmReturn);
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ NUdf::TUnboxedValue res = graph->GetValue();
+ UNIT_ASSERT(!res.IsSpecial());
+
+ NUdf::TUnboxedValue v;
+ auto status = res.Fetch(v);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Ok, status);
+
+ for (auto s : data) {
+ UNIT_ASSERT_C(v.Contains(NUdf::TUnboxedValue(MakeString(s))), s);
+ }
+ UNIT_ASSERT(!v.Contains(NUdf::TUnboxedValue(MakeString("green cucumber"))));
+
+ status = res.Fetch(v);
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, status);
+ };
+
+ for (auto hashed : {true, false}) {
+ for (auto multi : {true, false}) {
+ for (auto compact : {true, false}) {
+ for (auto withPayload : {true, false}) {
+ test(hashed, multi, compact, withPayload);
+ }
+ }
+ }
+ }
+ }
+#endif
+ template <bool LLVM>
+ static void TestDictWithDataKeyImpl(bool optionalKey, bool multi, bool compact, bool withNull, bool withData) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TType* keyType = pb.NewDataType(NUdf::EDataSlot::Int32, optionalKey);
+ TType* valueType = pb.NewDataType(NUdf::EDataSlot::Int32, false);
+ TType* tupleType = pb.NewTupleType({keyType, valueType});
+ TVector<TRuntimeNode> items;
+ TVector<TRuntimeNode> keys;
+ if (withNull) {
+ UNIT_ASSERT(optionalKey);
+ keys.push_back(pb.NewEmptyOptional(keyType));
+ for (size_t k = 0; k < 1 + multi; ++k) {
+ items.push_back(pb.NewTuple(tupleType, {keys.back(), pb.NewDataLiteral((i32)items.size())}));
+ }
+ }
+ if (withData) {
+ for (i32 i = 0; i < 2; ++i) {
+ auto key = pb.NewDataLiteral(i);
+ if (optionalKey) {
+ key = pb.NewOptional(key);
+ }
+ keys.push_back(key);
+ for (size_t k = 0; k < 1 + multi; ++k) {
+ items.push_back(pb.NewTuple(tupleType, {key, pb.NewDataLiteral((i32)items.size())}));
+ }
+ }
+ }
+ auto list = pb.NewList(tupleType, items);
+ auto keyList = pb.NewList(keyType, keys);
+ auto dict = pb.ToHashedDict(list, multi, [&](TRuntimeNode tuple) { return pb.Nth(tuple, 0); }, [&pb](TRuntimeNode tuple) { return pb.Nth(tuple, 1); }, compact);
+
+ auto compareLists = [&](bool itemIsTuple, TRuntimeNode list1, TRuntimeNode list2) {
+ return pb.And({
+ pb.Equals(
+ pb.Length(list1),
+ pb.Length(list2)
+ ),
+ pb.Not(
+ pb.Exists(
+ pb.Head(
+ pb.SkipWhile(
+ pb.Zip({list1, list2}),
+ [&](TRuntimeNode pair) {
+ if (itemIsTuple) {
+ return pb.And({
+ pb.AggrEquals(pb.Nth(pb.Nth(pair, 0), 0), pb.Nth(pb.Nth(pair, 1), 0)),
+ pb.AggrEquals(pb.Nth(pb.Nth(pair, 0), 1), pb.Nth(pb.Nth(pair, 1), 1)),
+ });
+ } else {
+ return pb.AggrEquals(pb.Nth(pair, 0), pb.Nth(pair, 1));
+ }
+ }
+ )
+ )
+ )
+ )
+ });
+ };
+
+ TVector<TRuntimeNode> results;
+
+ // Check Dict has items
+ results.push_back(pb.AggrEquals(
+ pb.HasItems(dict),
+ pb.NewDataLiteral(withNull || withData)
+ ));
+
+ // Check Dict length
+ results.push_back(pb.AggrEquals(
+ pb.Length(dict),
+ pb.NewDataLiteral((ui64)keys.size())
+ ));
+
+ // Check Dict Contains
+ results.push_back(pb.AllOf(
+ pb.Map(list, [&](TRuntimeNode tuple) {
+ return pb.Contains(dict, pb.Nth(tuple, 0));
+ }),
+ [&](TRuntimeNode item) { return item; }
+ ));
+
+ // Check Dict Lookup
+ results.push_back(compareLists(false,
+ pb.Sort(
+ pb.FlatMap(
+ pb.Map(
+ keyList,
+ [&](TRuntimeNode key) {
+ return pb.Unwrap(pb.Lookup(dict, key), pb.NewDataLiteral<NUdf::EDataSlot::String>("Lookup failed"), "", 0, 0);
+ }
+ ),
+ [&](TRuntimeNode item) {
+ return multi ? item : pb.NewOptional(item);
+ }
+ ),
+ pb.NewDataLiteral(true),
+ [&](TRuntimeNode item) { return item; }
+ ),
+ pb.Sort(
+ pb.Map(list, [&](TRuntimeNode tuple) {
+ return pb.Nth(tuple, 1);
+ }),
+ pb.NewDataLiteral(true),
+ [&](TRuntimeNode item) { return item; }
+ )
+ ));
+
+ // Check Dict items iterator
+ results.push_back(compareLists(true,
+ pb.Sort(
+ pb.FlatMap(
+ pb.DictItems(dict),
+ [&](TRuntimeNode pair) {
+ if (multi) {
+ return pb.Map(
+ pb.Nth(pair, 1),
+ [&](TRuntimeNode p) {
+ return pb.NewTuple({pb.Nth(pair, 0), p});
+ }
+ );
+ } else {
+ return pb.NewOptional(pair);
+ }
+ }
+ ),
+ pb.NewTuple({pb.NewDataLiteral(true), pb.NewDataLiteral(true)}),
+ [&](TRuntimeNode item) { return item; }
+ ),
+ list
+ ));
+
+ // Check Dict payloads iterator
+ results.push_back(compareLists(false,
+ pb.Sort(
+ pb.FlatMap(
+ pb.DictPayloads(dict),
+ [&](TRuntimeNode item) {
+ return multi ? item : pb.NewOptional(item);
+ }
+ ),
+ pb.NewDataLiteral(true),
+ [&](TRuntimeNode item) { return item; }
+ ),
+ pb.Map(
+ list,
+ [&](TRuntimeNode item) {
+ return pb.Nth(item, 1);
+ }
+ )
+ ));
+
+ auto graph = setup.BuildGraph(pb.NewTuple(results));
+ NUdf::TUnboxedValue res = graph->GetValue();
+
+ UNIT_ASSERT_C(res.GetElement(0).Get<bool>(), "Dict HasItems fail");
+ UNIT_ASSERT_C(res.GetElement(1).Get<bool>(), "Dict Length fail");
+ UNIT_ASSERT_C(res.GetElement(2).Get<bool>(), "Dict Contains fail");
+ UNIT_ASSERT_C(res.GetElement(3).Get<bool>(), "Dict Lookup fail");
+ UNIT_ASSERT_C(res.GetElement(4).Get<bool>(), "DictItems fail");
+ UNIT_ASSERT_C(res.GetElement(5).Get<bool>(), "DictPayloads fail");
+ }
+
+ Y_UNIT_TEST_LLVM(TestDictWithDataKey) {
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/false, /*compact*/false, /*withNull*/false, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/false, /*compact*/false, /*withNull*/false, /*withData*/false); // empty dict
+ }
+
+ Y_UNIT_TEST_LLVM(TestDictCompactWithDataKey) {
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/false, /*compact*/true, /*withNull*/false, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/false, /*compact*/true, /*withNull*/false, /*withData*/false); // empty dict
+ }
+
+ Y_UNIT_TEST_LLVM(TestDictMultiWithDataKey) {
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/true, /*compact*/false, /*withNull*/false, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/true, /*compact*/false, /*withNull*/false, /*withData*/false); // empty dict
+ }
+
+ Y_UNIT_TEST_LLVM(TestDictCompactMultiWithDataKey) {
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/true, /*compact*/true, /*withNull*/false, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*multi*/true, /*compact*/true, /*withNull*/false, /*withData*/false); // empty dict
+ }
+
+ Y_UNIT_TEST_LLVM(TestDictWithOptionalDataKey) {
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/false, /*withNull*/false, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/false, /*withNull*/true, /*withData*/false);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/false, /*withNull*/true, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/false, /*withNull*/false, /*withData*/false); // empty dict
+ }
+
+ Y_UNIT_TEST_LLVM(TestDictCompactWithOptionalDataKey) {
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/true, /*withNull*/false, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/true, /*withNull*/true, /*withData*/false);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/true, /*withNull*/true, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/false, /*compact*/true, /*withNull*/false, /*withData*/false); // empty dict
+ }
+
+ Y_UNIT_TEST_LLVM(TestDictMultiWithOptionalDataKey) {
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/false, /*withNull*/false, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/false, /*withNull*/true, /*withData*/false);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/false, /*withNull*/true, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/false, /*withNull*/false, /*withData*/false); // empty dict
+ }
+
+ Y_UNIT_TEST_LLVM(TestDictCompactMultiWithOptionalDataKey) {
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/true, /*withNull*/false, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/true, /*withNull*/true, /*withData*/false);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/true, /*withNull*/true, /*withData*/true);
+ TestDictWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*multi*/true, /*compact*/true, /*withNull*/false, /*withData*/false); // empty dict
+ }
+
+ template <bool LLVM>
+ static void TestSetWithDataKeyImpl(bool optionalKey, bool compact, bool withNull, bool withData) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TType* keyType = pb.NewDataType(NUdf::EDataSlot::Int32, optionalKey);
+ TVector<TRuntimeNode> keys;
+ if (withNull) {
+ UNIT_ASSERT(optionalKey);
+ keys.push_back(pb.NewEmptyOptional(keyType));
+ }
+ if (withData) {
+ for (i32 i = 0; i < 2; ++i) {
+ auto key = pb.NewDataLiteral(i);
+ if (optionalKey) {
+ key = pb.NewOptional(key);
+ }
+ keys.push_back(key);
+ }
+ }
+ auto keyList = pb.NewList(keyType, keys);
+ auto set = pb.ToHashedDict(keyList, false, [&](TRuntimeNode key) { return key; }, [&pb](TRuntimeNode) { return pb.NewVoid(); }, compact);
+
+ auto compareLists = [&](TRuntimeNode list1, TRuntimeNode list2) {
+ return pb.And({
+ pb.Equals(
+ pb.Length(list1),
+ pb.Length(list2)
+ ),
+ pb.Not(
+ pb.Exists(
+ pb.Head(
+ pb.SkipWhile(
+ pb.Zip({list1, list2}),
+ [&](TRuntimeNode pair) {
+ return pb.AggrEquals(pb.Nth(pair, 0), pb.Nth(pair, 1));
+ }
+ )
+ )
+ )
+ )
+ });
+ };
+
+ TVector<TRuntimeNode> results;
+
+ // Check Set has items
+ results.push_back(pb.AggrEquals(
+ pb.HasItems(set),
+ pb.NewDataLiteral(withNull || withData)
+ ));
+
+ // Check Set length
+ results.push_back(pb.AggrEquals(
+ pb.Length(set),
+ pb.NewDataLiteral((ui64)keys.size())
+ ));
+
+ // Check Set Contains
+ results.push_back(pb.AllOf(
+ pb.Map(keyList, [&](TRuntimeNode key) {
+ return pb.Contains(set, key);
+ }),
+ [&](TRuntimeNode item) { return item; }
+ ));
+
+ // Check Set Lookup
+ results.push_back(pb.AllOf(
+ pb.Map(keyList, [&](TRuntimeNode key) {
+ return pb.Exists(pb.Lookup(set, key));
+ }),
+ [&](TRuntimeNode item) { return item; }
+ ));
+
+ // Check Set items iterator
+ results.push_back(compareLists(
+ pb.Sort(
+ pb.DictKeys(set),
+ pb.NewDataLiteral(true),
+ [&](TRuntimeNode item) { return item; }
+ ),
+ keyList
+ ));
+
+ auto graph = setup.BuildGraph(pb.NewTuple(results));
+ NUdf::TUnboxedValue res = graph->GetValue();
+
+ UNIT_ASSERT_C(res.GetElement(0).Get<bool>(), "Set HasItems fail");
+ UNIT_ASSERT_C(res.GetElement(1).Get<bool>(), "Set Length fail");
+ UNIT_ASSERT_C(res.GetElement(2).Get<bool>(), "Set Contains fail");
+ UNIT_ASSERT_C(res.GetElement(3).Get<bool>(), "Set Lookup fail");
+ UNIT_ASSERT_C(res.GetElement(4).Get<bool>(), "Set DictKeys fail");
+ }
+
+ Y_UNIT_TEST_LLVM(TestSetWithDataKey) {
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*compact*/false, /*withNull*/false, /*withData*/true);
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*compact*/false, /*withNull*/false, /*withData*/false); // empty set
+ }
+
+ Y_UNIT_TEST_LLVM(TestSetCompactWithDataKey) {
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*compact*/true, /*withNull*/false, /*withData*/true);
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/false, /*compact*/true, /*withNull*/false, /*withData*/false); // empty set
+ }
+
+ Y_UNIT_TEST_LLVM(TestSetWithOptionalDataKey) {
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/false, /*withNull*/false, /*withData*/true);
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/false, /*withNull*/true, /*withData*/false);
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/false, /*withNull*/true, /*withData*/true);
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/false, /*withNull*/false, /*withData*/false); // empty set
+ }
+
+ Y_UNIT_TEST_LLVM(TestSetCompactWithOptionalDataKey) {
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/true, /*withNull*/false, /*withData*/true);
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/true, /*withNull*/true, /*withData*/false);
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/true, /*withNull*/true, /*withData*/true);
+ TestSetWithDataKeyImpl<LLVM>(/*optionalKey*/true, /*compact*/true, /*withNull*/false, /*withData*/false); // empty set
+ }
+}
+
+
+} // namespace
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp
new file mode 100644
index 00000000000..2a52f73ef33
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_variant_ut.cpp
@@ -0,0 +1,527 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+Y_UNIT_TEST_SUITE(TMiniKQLVariantTest) {
+ Y_UNIT_TEST_LLVM(TestGuessTuple) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)});
+ const auto varType = pb.NewVariantType(tupleType);
+ const auto var1 = pb.NewVariant(data1, 0, varType);
+ const auto var2 = pb.NewVariant(data2, 1, varType);
+ std::vector<TRuntimeNode> tupleItems;
+ tupleItems.push_back(pb.Guess(var1, 0));
+ tupleItems.push_back(pb.Guess(var1, 1));
+ tupleItems.push_back(pb.Guess(var2, 0));
+ tupleItems.push_back(pb.Guess(var2, 1));
+ const auto pgmReturn = pb.NewTuple(tupleItems);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ UNIT_ASSERT(res.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 1);
+ UNIT_ASSERT(!res.GetElement(1));
+ UNIT_ASSERT(!res.GetElement(2));
+ UNIT_ASSERT(res.GetElement(3));
+ UNBOXED_VALUE_STR_EQUAL(res.GetElement(3), "abc");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGuessTupleOpt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)});
+ const auto varType = pb.NewVariantType(tupleType);
+ const auto var1 = pb.NewVariant(data1, 0, varType);
+ const auto var2 = pb.NewVariant(data2, 1, varType);
+ const auto jvar1 = pb.NewOptional(var1);
+ const auto jvar2 = pb.NewOptional(var2);
+ const auto nothing = pb.NewEmptyOptional(pb.NewOptionalType(varType));
+
+ std::vector<TRuntimeNode> tupleItems;
+ tupleItems.push_back(pb.Guess(jvar1, 0));
+ tupleItems.push_back(pb.Guess(jvar1, 1));
+ tupleItems.push_back(pb.Guess(jvar2, 0));
+ tupleItems.push_back(pb.Guess(jvar2, 1));
+ tupleItems.push_back(pb.Guess(nothing, 0));
+ tupleItems.push_back(pb.Guess(nothing, 1));
+ const auto pgmReturn = pb.NewTuple(tupleItems);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ UNIT_ASSERT(res.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 1);
+ UNIT_ASSERT(!res.GetElement(1));
+ UNIT_ASSERT(!res.GetElement(2));
+ UNIT_ASSERT(res.GetElement(3));
+ UNBOXED_VALUE_STR_EQUAL(res.GetElement(3), "abc");
+ UNIT_ASSERT(!res.GetElement(4));
+ UNIT_ASSERT(!res.GetElement(5));
+ }
+
+ Y_UNIT_TEST_LLVM(TestGuessStruct) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<char*>::Id)}});
+ const auto varType = pb.NewVariantType(structType);
+ const auto var1 = pb.NewVariant(data1, "x", varType);
+ const auto var2 = pb.NewVariant(data2, "y", varType);
+ std::vector<TRuntimeNode> tupleItems;
+ tupleItems.push_back(pb.Guess(var1, "x"));
+ tupleItems.push_back(pb.Guess(var1, "y"));
+ tupleItems.push_back(pb.Guess(var2, "x"));
+ tupleItems.push_back(pb.Guess(var2, "y"));
+ const auto pgmReturn = pb.NewTuple(tupleItems);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ UNIT_ASSERT(res.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 1);
+ UNIT_ASSERT(!res.GetElement(1));
+ UNIT_ASSERT(!res.GetElement(2));
+ UNIT_ASSERT(res.GetElement(3));
+ UNBOXED_VALUE_STR_EQUAL(res.GetElement(3), "abc");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGuessStructOpt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<char*>::Id)}});
+ const auto varType = pb.NewVariantType(structType);
+ const auto var1 = pb.NewVariant(data1, "x", varType);
+ const auto var2 = pb.NewVariant(data2, "y", varType);
+ const auto jvar1 = pb.NewOptional(var1);
+ const auto jvar2 = pb.NewOptional(var2);
+ const auto nothing = pb.NewEmptyOptional(pb.NewOptionalType(varType));
+
+ std::vector<TRuntimeNode> tupleItems;
+ tupleItems.push_back(pb.Guess(jvar1, "x"));
+ tupleItems.push_back(pb.Guess(jvar1, "y"));
+ tupleItems.push_back(pb.Guess(jvar2, "x"));
+ tupleItems.push_back(pb.Guess(jvar2, "y"));
+ tupleItems.push_back(pb.Guess(nothing, "x"));
+ tupleItems.push_back(pb.Guess(nothing, "y"));
+ const auto pgmReturn = pb.NewTuple(tupleItems);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ UNIT_ASSERT(res.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(res.GetElement(0).template Get<ui32>(), 1);
+ UNIT_ASSERT(!res.GetElement(1));
+ UNIT_ASSERT(!res.GetElement(2));
+ UNIT_ASSERT(res.GetElement(3));
+ UNBOXED_VALUE_STR_EQUAL(res.GetElement(3), "abc");
+ UNIT_ASSERT(!res.GetElement(4));
+ UNIT_ASSERT(!res.GetElement(5));
+ }
+
+ Y_UNIT_TEST_LLVM(TestVisitAllTuple) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto at = pb.NewDataLiteral<NUdf::EDataSlot::String>("@");
+ const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)});
+ const auto varType = pb.NewVariantType(tupleType);
+ const auto var1 = pb.NewVariant(data1, 0, varType);
+ const auto var2 = pb.NewVariant(data2, 1, varType);
+ const auto list = pb.NewList(varType, {var1, var2});
+ const auto pgmReturn = pb.Map(list, [&](TRuntimeNode item) {
+ return pb.VisitAll(item, [&](ui32 index, TRuntimeNode item) {
+ if (!index) {
+ return pb.Concat(at, pb.ToString(item));
+ } else {
+ return pb.Concat(at, item);
+ }
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ UNBOXED_VALUE_STR_EQUAL(res.GetElement(0), "@1");
+ UNBOXED_VALUE_STR_EQUAL(res.GetElement(1), "@abc");
+ }
+
+ Y_UNIT_TEST_LLVM(TestVisitAllStruct) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto at = pb.NewDataLiteral<NUdf::EDataSlot::String>("@");
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<char*>::Id)}});
+ const auto varType = pb.NewVariantType(structType);
+ const auto var1 = pb.NewVariant(data1, "x", varType);
+ const auto var2 = pb.NewVariant(data2, "y", varType);
+ const auto list = pb.NewList(varType, {var1, var2});
+
+ const auto xIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("x");
+ const auto pgmReturn = pb.Map(list, [&](TRuntimeNode item) {
+ return pb.VisitAll(item, [&](ui32 index, TRuntimeNode item) {
+ if (xIndex == index) {
+ return pb.Concat(at, pb.ToString(item));
+ } else {
+ return pb.Concat(at, item);
+ }
+ });
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ UNBOXED_VALUE_STR_EQUAL(res.GetElement(0), "@1");
+ UNBOXED_VALUE_STR_EQUAL(res.GetElement(1), "@abc");
+ }
+
+ Y_UNIT_TEST_LLVM(TestVisitAllTupleFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto at = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("@");
+ const auto data1 = pb.NewDataLiteral<ui64>(3ULL);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("abc");
+ const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui64>::Id), pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)});
+ const auto varType = pb.NewVariantType(tupleType);
+ const auto var1 = pb.NewVariant(data1, 0, varType);
+ const auto var2 = pb.NewVariant(data2, 1, varType);
+ const auto list = pb.NewList(varType, {var1, var2});
+ const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(list), [&](TRuntimeNode item) {
+ return pb.VisitAll(item, [&](ui32 index, TRuntimeNode item) {
+ if (!index) {
+ return pb.ToFlow(pb.Replicate(at, item, __FILE__, __LINE__, 0U));
+ } else {
+ return pb.ToFlow(pb.NewOptional(item));
+ }
+ });
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item, "@");
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item, "@");
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item, "@");
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item, "abc");
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish);
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestVisitAllStructFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto at = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("@");
+ const auto data1 = pb.NewDataLiteral<ui64>(4ULL);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("abc");
+ const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui64>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)}});
+ const auto varType = pb.NewVariantType(structType);
+ const auto var1 = pb.NewVariant(data1, "x", varType);
+ const auto var2 = pb.NewVariant(data2, "y", varType);
+ const auto list = pb.NewList(varType, {var2, var1, var2});
+
+ const auto xIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("x");
+ const auto pgmReturn = pb.FromFlow(pb.FlatMap(pb.ToFlow(list), [&](TRuntimeNode item) {
+ return pb.VisitAll(item, [&](ui32 index, TRuntimeNode item) {
+ if (xIndex == index) {
+ return pb.ToFlow(pb.Replicate(at, item, __FILE__, __LINE__, 0U));
+ } else {
+ return pb.ToFlow(pb.NewOptional(item));
+ }
+ });
+ }));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item, "abc");
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item, "@");
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item, "@");
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item, "@");
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item, "@");
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item, "abc");
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish);
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestVisitAllStructWideFlow) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto at = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("@");
+ const auto data1 = pb.NewDataLiteral<ui64>(4ULL);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("abc");
+ const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui64>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)}});
+ const auto varType = pb.NewVariantType(structType);
+ const auto var1 = pb.NewVariant(data1, "x", varType);
+ const auto var2 = pb.NewVariant(data2, "y", varType);
+ const auto list = pb.NewList(varType, {var2, var1, var2});
+
+ const auto xIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("x");
+ const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.FlatMap(pb.ToFlow(list), [&](TRuntimeNode item) {
+ return pb.VisitAll(item, [&](ui32 index, TRuntimeNode item) {
+ if (xIndex == index) {
+ return pb.ExpandMap(pb.ToFlow(pb.Replicate(at, item, __FILE__, __LINE__, 0U)), [&](TRuntimeNode x) -> TRuntimeNode::TList { return {x, pb.NewDataLiteral(i32(index))}; });
+ } else {
+ return pb.ExpandMap(pb.ToFlow(pb.NewOptional(item)), [&](TRuntimeNode x) -> TRuntimeNode::TList { return {x, pb.NewDataLiteral(-i32(index))}; });
+ }
+ });
+ }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto res = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "abc");
+ UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), -1);
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "@");
+ UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), 0);
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "@");
+ UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), 0);
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "@");
+ UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), 0);
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "@");
+ UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), 0);
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Ok);
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "abc");
+ UNIT_ASSERT_EQUAL(item.GetElement(1).Get<i32>(), -1);
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish);
+ UNIT_ASSERT_EQUAL(res.Fetch(item), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestWayTuple) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)});
+ const auto varType = pb.NewVariantType(tupleType);
+ const auto var1 = pb.NewVariant(data1, 0, varType);
+ const auto var2 = pb.NewVariant(data2, 1, varType);
+ const auto list = pb.NewList(varType, {var2, var1});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) { return pb.Way(item); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestWayTupleOpt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto tupleType = pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id)});
+ const auto varType = pb.NewVariantType(tupleType);
+ const auto var1 = pb.NewVariant(data1, 0, varType);
+ const auto var2 = pb.NewVariant(data2, 1, varType);
+ const auto jvar1 = pb.NewOptional(var1);
+ const auto jvar2 = pb.NewOptional(var2);
+ const auto optType = pb.NewOptionalType(varType);
+ const auto nothing = pb.NewEmptyOptional(optType);
+ const auto list = pb.NewList(optType, {jvar2, nothing, jvar1});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) { return pb.Way(item); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 1U);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), 0U);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestWayStruct) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<char*>::Id)}});
+ const auto varType = pb.NewVariantType(structType);
+ const auto var1 = pb.NewVariant(data1, "x", varType);
+ const auto var2 = pb.NewVariant(data2, "y", varType);
+ const auto list = pb.NewList(varType, {var2, var1});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) { return pb.Way(item); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "y");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "x");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestWayStructOpt) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data1 = pb.NewDataLiteral<ui32>(1);
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("abc");
+ const auto structType = pb.NewStructType({{"x", pb.NewDataType(NUdf::TDataType<ui32>::Id)}, {"y", pb.NewDataType(NUdf::TDataType<char*>::Id)}});
+ const auto varType = pb.NewVariantType(structType);
+ const auto var1 = pb.NewVariant(data1, "x", varType);
+ const auto var2 = pb.NewVariant(data2, "y", varType);
+ const auto jvar1 = pb.NewOptional(var1);
+ const auto jvar2 = pb.NewOptional(var2);
+ const auto optType = pb.NewOptionalType(varType);
+ const auto nothing = pb.NewEmptyOptional(optType);
+ const auto list = pb.NewList(optType, {jvar2, nothing, jvar1});
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) { return pb.Way(item); }
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "y");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "x");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestItemInMap) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto varType = pb.NewVariantType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id), pb.NewDataType(NUdf::TDataType<bool>::Id)}));
+
+ const auto data0 = pb.NewVariant(pb.NewDataLiteral<i32>(77), 0, varType);
+ const auto data1 = pb.NewVariant(pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"), 1, varType);
+ const auto data2 = pb.NewVariant(pb.NewDataLiteral<bool>(false), 2, varType);
+ const auto data3 = pb.NewVariant(pb.NewDataLiteral<bool>(true), 2, varType);
+ const auto data4 = pb.NewVariant(pb.NewDataLiteral<NUdf::EDataSlot::String>("DEF"), 1, varType);
+ const auto data5 = pb.NewVariant(pb.NewDataLiteral<i32>(-1267), 0, varType);
+ const auto list = pb.NewList(varType, {data0, data1, data2, data3, data4, data5});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.VariantItem(item);
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 77);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(item.AsStringRef()), "abc");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(item.AsStringRef()), "DEF");
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1267);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestGuessInMap) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto varType = pb.NewVariantType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i32>::Id), pb.NewDataType(NUdf::TDataType<char*>::Id), pb.NewDataType(NUdf::TDataType<bool>::Id)}));
+
+ const auto data0 = pb.NewVariant(pb.NewDataLiteral<i32>(77), 0, varType);
+ const auto data1 = pb.NewVariant(pb.NewDataLiteral<NUdf::EDataSlot::String>("abc"), 1, varType);
+ const auto data2 = pb.NewVariant(pb.NewDataLiteral<bool>(false), 2, varType);
+ const auto data3 = pb.NewVariant(pb.NewDataLiteral<bool>(true), 2, varType);
+ const auto data4 = pb.NewVariant(pb.NewDataLiteral<NUdf::EDataSlot::String>("DEF"), 1, varType);
+ const auto data5 = pb.NewVariant(pb.NewDataLiteral<i32>(-1267), 0, varType);
+ const auto list = pb.NewList(varType, {data0, data1, data2, data3, data4, data5});
+
+ const auto pgmReturn = pb.Map(list,
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Guess(item, 0), pb.Guess(item, 2)});
+ });
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 77);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<bool>(), false);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<bool>(), true);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -1267);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp
new file mode 100644
index 00000000000..e5ac540ca0e
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chain_map_ut.cpp
@@ -0,0 +1,316 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 23u
+Y_UNIT_TEST_SUITE(TMiniKQLWideChain1MapTest) {
+ Y_UNIT_TEST_LLVM(TestThinLambda) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode) -> TRuntimeNode::TList { return {}; }),
+ [&](TRuntimeNode::TList inputs) { return inputs; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList outputs) { return outputs; }),
+ [&](TRuntimeNode::TList) -> TRuntimeNode { return pb.NewTuple({}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSimpleSwap) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList inputs) { return inputs; },
+ [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList { return {inputs.back(), outputs[1U], inputs.front()}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -2);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -3);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSimpleChain) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(-5)), pb.NewOptional(pb.NewDataLiteral<i32>(-6))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-4)), pb.NewOptional(pb.NewDataLiteral<i32>(-7))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-7)), pb.NewOptional(pb.NewDataLiteral<i32>(-8))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-9))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList inputs) -> TRuntimeNode::TList { return {pb.Add(inputs.front(), inputs[1U]), pb.NewEmptyOptional(dataType), pb.Sub(inputs.back(), inputs[1U])}; },
+ [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList {
+ return {pb.AggrAdd(outputs.back(), inputs[1U]), outputs.front(), pb.Decrement(outputs[1])};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -4);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -1);
+ UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -5);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -4);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -7);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -5);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -5);
+ UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -5);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -7);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -6);
+ UNIT_ASSERT(NUdf::EFetchStatus::Finish == iterator.Fetch(item));
+ UNIT_ASSERT(NUdf::EFetchStatus::Finish == iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestAgrregateWithPrevious) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(5)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList inputs) -> TRuntimeNode::TList { return {inputs[1U], inputs[2U], inputs[0U]}; },
+ [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList {
+ return {pb.AggrMin(inputs[0U], outputs[1U]), pb.AggrMax(inputs[1U], outputs[2U]), pb.AggrAdd(outputs[0U], inputs[2U])};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 5);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestPasstroughtFieldSplitAsIs) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(-6)), pb.NewOptional(pb.NewDataLiteral<i32>(-5))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-4)), pb.NewOptional(pb.NewDataLiteral<i32>(-4))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-7)), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(0))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList inputs) -> TRuntimeNode::TList { return {inputs[1U], pb.Mul(inputs.front(), inputs.back()), inputs[1U]}; },
+ [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList {
+ return {inputs[1U], pb.Mul(outputs[1U], pb.Add(inputs.back(), inputs.front())), inputs[1U]};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -6);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -5);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -6);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 10);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -7);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 0);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFieldBothWayPasstroughtAndArg) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(-5)), pb.NewOptional(pb.NewDataLiteral<i32>(-6))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-4)), pb.NewOptional(pb.NewDataLiteral<i32>(-7))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-7)), pb.NewOptional(pb.NewDataLiteral<i32>(-8))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-9))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList inputs) -> TRuntimeNode::TList { return {inputs[1U], pb.Sub(inputs.front(), inputs.back()), pb.Minus(inputs[1U])}; },
+ [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList {
+ return {inputs[1U], pb.Sub(outputs[1U], pb.Add(inputs.back(), inputs.front())), pb.Minus(inputs[1U])};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -5);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 7);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 5);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 12);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -7);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 17);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 22);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDotCalculateUnusedField) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+ const auto null = pb.NewEmptyOptional(dataType);
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), null, pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), null, pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), null, pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), null, pb.NewOptional(pb.NewDataLiteral<i32>(-4))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::Utf8>("Veszély! Aknák!");
+
+ const auto pgmReturn = pb.FromFlow(pb.NarrowMap(pb.WideChain1Map(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList inputs) -> TRuntimeNode::TList { return {inputs.back(), pb.Unwrap(inputs[1U], landmine, __FILE__, __LINE__, 0), inputs.front()}; },
+ [&](TRuntimeNode::TList inputs, TRuntimeNode::TList outputs) -> TRuntimeNode::TList {
+ return {pb.Mul(outputs.front(), inputs.back()), pb.Unwrap(inputs[1U], landmine, __FILE__, __LINE__, 0), pb.Add(inputs.front(), outputs.back())};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items.back(), items.front()}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1);
+ UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 6);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -6);
+ UNIT_ASSERT(NUdf::EFetchStatus::Ok == iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 10);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 24);
+ UNIT_ASSERT(NUdf::EFetchStatus::Finish == iterator.Fetch(item));
+ UNIT_ASSERT(NUdf::EFetchStatus::Finish == iterator.Fetch(item));
+ }
+}
+#endif
+}
+}
+
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp
new file mode 100644
index 00000000000..b2865328e86
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_chopper_ut.cpp
@@ -0,0 +1,474 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u
+Y_UNIT_TEST_SUITE(TMiniKQLWideChopperTest) {
+ Y_UNIT_TEST_LLVM(TestConcatKeyToItems) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {pb.Substring(items.front(), pb.Sub(pb.Size(items.front()), pb.NewDataLiteral<ui32>(4U)), pb.NewDataLiteral<ui32>(4U))};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) {
+ return pb.AggrNotEquals(keys.front(), items.front());
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode input) {
+ return pb.WideMap(input, [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {pb.AggrConcat(items.back(), keys.front())};
+ });
+ }),
+ [&](TRuntimeNode::TList items) { return items.front(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 1 one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 2 two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 3 two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 4 one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 5 two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 6 two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 7 two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 8 two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 9 two");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCollectKeysOnly) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items.front()};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) {
+ return pb.AggrNotEquals(keys.front(), items.front());
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode) {
+ return pb.ExpandMap(pb.ToFlow(pb.NewOptional(keys.front())), [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ return {item};
+ } );
+ }),
+ [&](TRuntimeNode::TList items) { return items.front(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long key one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long key two");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestGetPart) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+ const auto data5 = pb.NewTuple(tupleType, {longKeyOne, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyOne, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyOne, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyOne, value8});
+
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items.front()};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) {
+ return pb.AggrNotEquals(keys.front(), items.front());
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode input) {
+ return pb.Take(pb.Skip(input, pb.NewDataLiteral<ui64>(1ULL)), pb.NewDataLiteral<ui64>(3ULL));
+ }),
+ [&](TRuntimeNode::TList items) { return items.back(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 3");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 5");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 6");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 7");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSwitchByBoolFieldAndDontUseKey) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ const auto tupleType = pb.NewTupleType({pb.NewOptionalType(dataType), dataType, boolType});
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewOptionalType(dataType));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("one"));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("two"));
+
+ const auto trueVal = pb.NewDataLiteral<bool>(true);
+ const auto falseVal = pb.NewDataLiteral<bool>(false);
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {key0, value1, trueVal});
+ const auto data2 = pb.NewTuple(tupleType, {key1, value2, falseVal});
+ const auto data3 = pb.NewTuple(tupleType, {key2, value3, falseVal});
+ const auto data4 = pb.NewTuple(tupleType, {key0, value4, trueVal});
+ const auto data5 = pb.NewTuple(tupleType, {key1, value5, falseVal});
+ const auto data6 = pb.NewTuple(tupleType, {key2, value6, falseVal});
+ const auto data7 = pb.NewTuple(tupleType, {key0, value7, falseVal});
+ const auto data8 = pb.NewTuple(tupleType, {key1, value8, falseVal});
+ const auto data9 = pb.NewTuple(tupleType, {key2, value9, trueVal});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {pb.Unwrap(items.front(), landmine, __FILE__, __LINE__, 0)};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) {
+ return items.back();
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode input) {
+ return pb.Take(input, pb.NewDataLiteral<ui64>(2ULL));
+ }),
+ [&](TRuntimeNode::TList items) { return items[1U]; }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 1");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 2");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 4");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 5");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 9");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCollectKeysIfPresent) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ const auto tupleType = pb.NewTupleType({pb.NewOptionalType(dataType), dataType, boolType});
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewOptionalType(dataType));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("one"));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("two"));
+
+ const auto trueVal = pb.NewDataLiteral<bool>(true);
+ const auto falseVal = pb.NewDataLiteral<bool>(false);
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {key1, value1, trueVal});
+ const auto data2 = pb.NewTuple(tupleType, {key1, value2, falseVal});
+ const auto data3 = pb.NewTuple(tupleType, {key1, value3, falseVal});
+ const auto data4 = pb.NewTuple(tupleType, {key0, value4, trueVal});
+ const auto data5 = pb.NewTuple(tupleType, {key0, value5, falseVal});
+ const auto data6 = pb.NewTuple(tupleType, {key2, value6, falseVal});
+ const auto data7 = pb.NewTuple(tupleType, {key0, value7, falseVal});
+ const auto data8 = pb.NewTuple(tupleType, {key0, value8, falseVal});
+ const auto data9 = pb.NewTuple(tupleType, {key0, value9, trueVal});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items.front()};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) {
+ return pb.AggrNotEquals(keys.front(), items.front());
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode part) {
+ return pb.IfPresent(keys,
+ [&](TRuntimeNode::TList keys) { return pb.ExpandMap(pb.ToFlow(pb.NewList(dataType, keys)), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } ); },
+ pb.WideMap(part, [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items[1U]}; } ));
+ }),
+ [&](TRuntimeNode::TList items) { return items.front(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 4");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 5");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 7");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 8");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 9");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestConditionalByKeyPart) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ const auto tupleType = pb.NewTupleType({pb.NewOptionalType(dataType), dataType, boolType});
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewOptionalType(dataType));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("one"));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("two"));
+
+ const auto trueVal = pb.NewDataLiteral<bool>(true);
+ const auto falseVal = pb.NewDataLiteral<bool>(false);
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {key1, value1, trueVal});
+ const auto data2 = pb.NewTuple(tupleType, {key1, value2, trueVal});
+ const auto data3 = pb.NewTuple(tupleType, {key1, value3, falseVal});
+ const auto data4 = pb.NewTuple(tupleType, {key1, value4, falseVal});
+ const auto data5 = pb.NewTuple(tupleType, {key2, value5, falseVal});
+ const auto data6 = pb.NewTuple(tupleType, {key2, value6, falseVal});
+ const auto data7 = pb.NewTuple(tupleType, {key2, value7, trueVal});
+ const auto data8 = pb.NewTuple(tupleType, {key0, value8, trueVal});
+ const auto data9 = pb.NewTuple(tupleType, {key0, value9, falseVal});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items.front(), items.back()};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) {
+ return pb.Or({pb.AggrNotEquals(keys.front(), items.front()), pb.AggrNotEquals(keys.back(), items.back())});
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode part) {
+ return pb.If(keys.back(),
+ pb.ExpandMap(pb.ToFlow(keys.front()), [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; } ),
+ pb.WideMap(part, [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items[1U]}; } ));
+ }),
+ [&](TRuntimeNode::TList items) { return items.front(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 3");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 4");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 5");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 6");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 9");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestThinAllLambdas) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto tupleType = pb.NewTupleType({});
+
+
+ const auto data = pb.NewTuple({});
+
+ const auto list = pb.NewList(tupleType, {data, data, data, data});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideChopper(pb.ExpandMap(pb.ToFlow(list),
+ [](TRuntimeNode) -> TRuntimeNode::TList { return {}; }),
+ [](TRuntimeNode::TList items) { return items; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList) { return pb.NewDataLiteral<bool>(true); },
+ [&](TRuntimeNode::TList, TRuntimeNode input) { return pb.WideMap(input, [](TRuntimeNode::TList items) { return items; }); }),
+ [&](TRuntimeNode::TList) { return pb.NewTuple({}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+#endif
+}
+}
+
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp
new file mode 100644
index 00000000000..6d8a18901b1
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_combine_ut.cpp
@@ -0,0 +1,1699 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+#include <cstring>
+#include <random>
+#include <algorithm>
+
+namespace NKikimr {
+namespace NMiniKQL {
+namespace {
+const auto border = 9124596000000000ULL;
+}
+
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u
+Y_UNIT_TEST_SUITE(TMiniKQLWideCombinerTest) {
+ Y_UNIT_TEST_LLVM(TestLongStringsRefCounting) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto optionalType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {pb.NewOptional(items.back()), pb.NewOptional(keys.front()), pb.NewEmptyOptional(optionalType), pb.NewEmptyOptional(optionalType)};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.NewOptional(items.back()), state.front(), state[1U], state[2U]};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ state.erase(state.cbegin());
+ return {pb.FlatMap(pb.NewList(optionalType, state), [&](TRuntimeNode item) { return item; } )};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return pb.Fold1(items.front(),
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item);
+ }
+ );
+ }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 2 / key two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long key one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 8 / very long value 7 / very long value 6");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestLongStringsPasstroughtRefCounting) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items.back(), keys.front(), items.back(), items.front()};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {items.back(), keys.front(), state[2U], state.back()};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return state;
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return pb.Fold1(pb.NewList(dataType, items),
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item);
+ }
+ );
+ }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 1 / key one / very long value 1 / key one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 3 / key two / very long value 2 / key two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 4 / very long key one / very long value 4 / very long key one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 9 / very long key two / very long value 5 / very long key two");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoNotCalculateUnusedInput) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto optionalType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, optionalType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5");
+
+ const auto empty = pb.NewDataLiteral<NUdf::EDataSlot::String>("");
+
+ const auto none = pb.NewEmptyOptional(optionalType);
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, none, value1});
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, none, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, none, value3});
+ const auto data4 = pb.NewTuple(tupleType, {keyOne, none, value4});
+ const auto data5 = pb.NewTuple(tupleType, {keyOne, none, value5});
+ const auto data6 = pb.NewTuple(tupleType, {keyOne, none, value1});
+ const auto data7 = pb.NewTuple(tupleType, {keyOne, none, value2});
+ const auto data8 = pb.NewTuple(tupleType, {keyTwo, none, value3});
+ const auto data9 = pb.NewTuple(tupleType, {keyTwo, none, value4});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Unwrap(pb.Nth(item, 1U), landmine, __FILE__, __LINE__, 0), pb.Nth(item, 2U)}; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items.back(), keys.front(), empty, empty};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {items.back(), state.front(), state[1U], state[2U]};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ state.insert(state.cbegin(), keys.cbegin(), keys.cend());
+ return {pb.NewList(dataType, state)};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return pb.Fold1(items.front(),
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item);
+ }
+ );
+ }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key one / value 2 / value 1 / value 5 / value 4");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key two / value 4 / value 3 / value 3 / value 2");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoNotCalculateUnusedOutput) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto optionalType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, optionalType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5");
+
+ const auto empty = pb.NewDataLiteral<NUdf::EDataSlot::String>("");
+
+ const auto none = pb.NewEmptyOptional(optionalType);
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, none, value1});
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, none, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, none, value3});
+ const auto data4 = pb.NewTuple(tupleType, {keyOne, none, value4});
+ const auto data5 = pb.NewTuple(tupleType, {keyOne, none, value5});
+ const auto data6 = pb.NewTuple(tupleType, {keyOne, none, value1});
+ const auto data7 = pb.NewTuple(tupleType, {keyOne, none, value2});
+ const auto data8 = pb.NewTuple(tupleType, {keyTwo, none, value3});
+ const auto data9 = pb.NewTuple(tupleType, {keyTwo, none, value4});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items[1U], items.back()};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.Concat(state.front(), items[1U]), pb.AggrConcat(pb.AggrConcat(state.back(), pb.NewDataLiteral<NUdf::EDataSlot::String>(", ")), items.back())};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.Unwrap(state.front(), landmine, __FILE__, __LINE__, 0), pb.AggrConcat(pb.AggrConcat(keys.front(), pb.NewDataLiteral<NUdf::EDataSlot::String>(": ")), state.back())};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.back(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key one: value 1, value 4, value 5, value 1, value 2");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key two: value 2, value 3, value 3, value 4");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestThinAllLambdas) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto tupleType = pb.NewTupleType({});
+ const auto data = pb.NewTuple({});
+
+ const auto list = pb.NewList(tupleType, {data, data, data, data});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(list),
+ [](TRuntimeNode) -> TRuntimeNode::TList { return {}; }), 0ULL,
+ [](TRuntimeNode::TList items) { return items; },
+ [](TRuntimeNode::TList, TRuntimeNode::TList items) { return items; },
+ [](TRuntimeNode::TList, TRuntimeNode::TList, TRuntimeNode::TList state) { return state; },
+ [](TRuntimeNode::TList, TRuntimeNode::TList state) { return state; }),
+ [&](TRuntimeNode::TList) { return pb.NewTuple({}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLWideCombinerPerfTest) {
+ Y_UNIT_TEST_LLVM(TestSumDoubleBooleanKeys) {
+ TSetup<LLVM> setup;
+
+ double positive = 0.0, negative = 0.0;
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ (sample.second > 0.0 ? positive : negative) += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.AggrGreater(items.front(), pb.NewDataLiteral(0.0))}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return items; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.front())}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { return state; }),
+ [&](TRuntimeNode::TList items) { return items.front(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); });
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto first = value.GetElement(0);
+ const auto second = value.GetElement(1);
+ const auto t2 = TInstant::Now();
+
+ if (first.template Get<double>() > 0.0) {
+ UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), positive);
+ UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), negative);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), negative);
+ UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), positive);
+ }
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleBooleanKeys) {
+ TSetup<LLVM> setup;
+
+ double pSum = 0.0, nSum = 0.0, pMax = 0.0, nMax = -1000.0, pMin = 1000.0, nMin = 0.0;
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ if (sample.second > 0.0) {
+ pSum += sample.second;
+ pMax = std::max(pMax, sample.second);
+ pMin = std::min(pMin, sample.second);
+ } else {
+ nSum += sample.second;
+ nMax = std::max(nMax, sample.second);
+ nMin = std::min(nMin, sample.second);
+ }
+ }
+
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.AggrGreater(items.front(), pb.NewDataLiteral(0.0))}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front(), items.front(), items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.AggrAdd(state.front(), items.front()), pb.AggrMin(state[1U], items.front()), pb.AggrMax(state.back(), items.back()) };
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { return state; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); });
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto first = value.GetElement(0);
+ const auto second = value.GetElement(1);
+ const auto t2 = TInstant::Now();
+
+ if (first.GetElement(0).template Get<double>() > 0.0) {
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), pSum);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), pMin);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), pMax);
+
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), nSum);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), nMin);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), nMax);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), nSum);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), nMin);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), nMax);
+
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), pSum);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), pMin);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), pMax);
+ }
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleSmallKey) {
+ TSetup<LLVM> setup;
+
+ std::unordered_map<i8, double> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ expects.emplace(sample.first, 0.0).first->second += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<i8, double>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back())}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), state.front()}; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ for (const auto& sample : I8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), ptr[i].GetElement(1).template Get<double>());
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleSmallKey) {
+ TSetup<LLVM> setup;
+
+ std::unordered_map<i8, std::array<double, 3U>> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, std::numeric_limits<double>::max(), std::numeric_limits<double>::min()}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<i8, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back())}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { state.insert(state.cbegin(), keys.front()); return state; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ for (const auto& sample : I8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleStringKey) {
+ TSetup<LLVM> setup;
+
+ std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size());
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); });
+
+ std::unordered_map<std::string, double> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : stringI8Samples) {
+ expects.emplace(sample.first, 0.0).first->second += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::string_view, double>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back())}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), state.front()}; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items));
+ for (const auto& sample : stringI8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElements()->AsStringRef(), ptr[i].GetElement(1).template Get<double>());
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleStringKey) {
+ TSetup<LLVM> setup;
+
+ std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size());
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); });
+
+ std::unordered_map<std::string, std::array<double, 3U>> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : stringI8Samples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::string_view, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back())}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { state.insert(state.cbegin(), keys.front()); return state; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items));
+ for (const auto& sample : stringI8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElements()->AsStringRef(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumTupleKey) {
+ TSetup<LLVM> setup;
+
+ std::vector<std::pair<std::pair<ui32, std::string>, double>> pairSamples(Ui16Samples.size());
+ std::transform(Ui16Samples.cbegin(), Ui16Samples.cend(), pairSamples.begin(), [](std::pair<ui16, double> src){ return std::make_pair(std::make_pair(ui32(src.first / 10U % 100U), ToString(src.first % 10U)), src.second); });
+
+ struct TPairHash { size_t operator()(const std::pair<ui32, std::string>& p) const { return CombineHashes(std::hash<ui32>()(p.first), std::hash<std::string_view>()(p.second)); } };
+
+ std::unordered_map<std::pair<ui32, std::string>, std::array<double, 3U>, TPairHash> expects;
+ const auto t = TInstant::Now();
+ for (const auto& sample : pairSamples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::pair<ui32, std::string>, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<const char*>::Id)}), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(pb.Nth(item, 0U), 0U), pb.Nth(pb.Nth(item, 0U), 1U), pb.Nth(item, 1U) }; }), 0ULL,
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front(), items[1U]}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back()) };
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), keys.back(), state.front(), state[1U], state.back()}; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple({pb.NewTuple({items[0U], items[1U]}), items[2U], items[3U], items[4U]}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(pairSamples.size(), items));
+ for (const auto& sample : pairSamples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ NUdf::TUnboxedValue* keys = nullptr;
+ pair[0] = graph->GetHolderFactory().CreateDirectArrayHolder(2U, keys);
+ keys[0] = NUdf::TUnboxedValuePod(sample.first.first);
+ keys[1] = NUdf::TUnboxedValuePod::Embedded(sample.first.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ const auto elements = ptr[i].GetElements();
+ two.emplace_back(std::make_pair(elements[0].GetElement(0).template Get<ui32>(), (elements[0].GetElements()[1]).AsStringRef()), std::array<double, 3U>{elements[1].template Get<double>(), elements[2].template Get<double>(), elements[3].template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestTpch) {
+ TSetup<LLVM> setup;
+
+ struct TPairHash { size_t operator()(const std::pair<std::string_view, std::string_view>& p) const { return CombineHashes(std::hash<std::string_view>()(p.first), std::hash<std::string_view>()(p.second)); } };
+
+ std::unordered_map<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>, TPairHash> expects;
+ const auto t = TInstant::Now();
+ for (auto& sample : TpchSamples) {
+ if (std::get<0U>(sample) <= border) {
+ const auto& ins = expects.emplace(std::pair<std::string_view, std::string_view>{std::get<1U>(sample), std::get<2U>(sample)}, std::pair<ui64, std::array<double, 5U>>{0ULL, {0., 0., 0., 0., 0.}});
+ auto& item = ins.first->second;
+ ++item.first;
+ std::get<0U>(item.second) += std::get<3U>(sample);
+ std::get<1U>(item.second) += std::get<5U>(sample);
+ std::get<2U>(item.second) += std::get<6U>(sample);
+ const auto v = std::get<3U>(sample) * (1. - std::get<5U>(sample));
+ std::get<3U>(item.second) += v;
+ std::get<4U>(item.second) += v * (1. + std::get<4U>(sample));
+ }
+ }
+ for (auto& item : expects) {
+ std::get<1U>(item.second.second) /= item.second.first;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::pair<std::string, std::string>, std::pair<ui64, std::array<double, 5U>>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui64>::Id),
+ pb.NewDataType(NUdf::TDataType<const char*>::Id),
+ pb.NewDataType(NUdf::TDataType<const char*>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id)
+ }));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCombiner(
+ pb.WideFilter(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U), pb.Nth(item, 4U), pb.Nth(item, 5U), pb.Nth(item, 6U)}; }),
+ [&](TRuntimeNode::TList items) { return pb.AggrLessOrEqual(items.front(), pb.NewDataLiteral<ui64>(border)); }
+ ), 0ULL,
+ [&](TRuntimeNode::TList item) -> TRuntimeNode::TList { return {item[1U], item[2U]}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ const auto price = items[3U];
+ const auto disco = items[5U];
+ const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco));
+ return {pb.NewDataLiteral<ui64>(1ULL), price, disco, items[6U], v, pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), items[4U]))};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ const auto price = items[3U];
+ const auto disco = items[5U];
+ const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco));
+ return {pb.Increment(state[0U]), pb.AggrAdd(state[1U], price), pb.AggrAdd(state[2U], disco), pb.AggrAdd(state[3U], items[6U]), pb.AggrAdd(state[4U], v), pb.AggrAdd(state[5U], pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), items[4U])))};
+ },
+ [&](TRuntimeNode::TList key, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {key.front(), key.back(), state[0U], state[1U], pb.Div(state[2U], state[0U]), state[3U], state[4U], state[5U]}; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(TpchSamples.size(), items));
+ for (const auto& sample : TpchSamples) {
+ NUdf::TUnboxedValue* elements = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(7U, elements);
+ elements[0] = NUdf::TUnboxedValuePod(std::get<0U>(sample));
+ elements[1] = NUdf::TUnboxedValuePod::Embedded(std::get<1U>(sample));
+ elements[2] = NUdf::TUnboxedValuePod::Embedded(std::get<2U>(sample));
+ elements[3] = NUdf::TUnboxedValuePod(std::get<3U>(sample));
+ elements[4] = NUdf::TUnboxedValuePod(std::get<4U>(sample));
+ elements[5] = NUdf::TUnboxedValuePod(std::get<5U>(sample));
+ elements[6] = NUdf::TUnboxedValuePod(std::get<6U>(sample));
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ const auto elements = ptr[i].GetElements();
+ two.emplace_back(std::make_pair(elements[0].AsStringRef(), elements[1].AsStringRef()), std::pair<ui64, std::array<double, 5U>>{elements[2].template Get<ui64>(), {elements[3].template Get<double>(), elements[4].template Get<double>(), elements[5].template Get<double>(), elements[6].template Get<double>(), elements[7].template Get<double>()}});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+}
+#endif
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 29u
+Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerTest) {
+ Y_UNIT_TEST_LLVM(TestLongStringsRefCounting) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto optionalType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {pb.NewOptional(items.back()), pb.NewOptional(keys.front()), pb.NewEmptyOptional(optionalType), pb.NewEmptyOptional(optionalType)};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.NewOptional(items.back()), state.front(), state[1U], state[2U]};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ state.erase(state.cbegin());
+ return {pb.FlatMap(pb.NewList(optionalType, state), [&](TRuntimeNode item) { return item; } )};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return pb.Fold1(items.front(),
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item);
+ }
+ );
+ }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 2 / key two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long key one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 8 / very long value 7 / very long value 6");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestLongStringsPasstroughtRefCounting) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items.back(), keys.front(), items.back(), items.front()};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {items.back(), keys.front(), state[2U], state.back()};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return state;
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return pb.Fold1(pb.NewList(dataType, items),
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item);
+ }
+ );
+ }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 1 / key one / very long value 1 / key one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 3 / key two / very long value 2 / key two");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 4 / very long key one / very long value 4 / very long key one");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long value 9 / very long key two / very long value 5 / very long key two");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoNotCalculateUnusedInput) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto optionalType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, optionalType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5");
+
+ const auto empty = pb.NewDataLiteral<NUdf::EDataSlot::String>("");
+
+ const auto none = pb.NewEmptyOptional(optionalType);
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, none, value1});
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, none, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, none, value3});
+ const auto data4 = pb.NewTuple(tupleType, {keyOne, none, value4});
+ const auto data5 = pb.NewTuple(tupleType, {keyOne, none, value5});
+ const auto data6 = pb.NewTuple(tupleType, {keyOne, none, value1});
+ const auto data7 = pb.NewTuple(tupleType, {keyOne, none, value2});
+ const auto data8 = pb.NewTuple(tupleType, {keyTwo, none, value3});
+ const auto data9 = pb.NewTuple(tupleType, {keyTwo, none, value4});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Unwrap(pb.Nth(item, 1U), landmine, __FILE__, __LINE__, 0), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items.back(), keys.front(), empty, empty};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {items.back(), state.front(), state[1U], state[2U]};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ state.insert(state.cbegin(), keys.cbegin(), keys.cend());
+ return {pb.NewList(dataType, state)};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return pb.Fold1(items.front(),
+ [&](TRuntimeNode item) { return item; },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pb.AggrConcat(pb.AggrConcat(state, pb.NewDataLiteral<NUdf::EDataSlot::String>(" / ")), item);
+ }
+ );
+ }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key one / value 2 / value 1 / value 5 / value 4");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key two / value 4 / value 3 / value 3 / value 2");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoNotCalculateUnusedOutput) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto optionalType = pb.NewOptionalType(dataType);
+ const auto tupleType = pb.NewTupleType({dataType, optionalType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5");
+
+ const auto empty = pb.NewDataLiteral<NUdf::EDataSlot::String>("");
+
+ const auto none = pb.NewEmptyOptional(optionalType);
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, none, value1});
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, none, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, none, value3});
+ const auto data4 = pb.NewTuple(tupleType, {keyOne, none, value4});
+ const auto data5 = pb.NewTuple(tupleType, {keyOne, none, value5});
+ const auto data6 = pb.NewTuple(tupleType, {keyOne, none, value1});
+ const auto data7 = pb.NewTuple(tupleType, {keyOne, none, value2});
+ const auto data8 = pb.NewTuple(tupleType, {keyTwo, none, value3});
+ const auto data9 = pb.NewTuple(tupleType, {keyTwo, none, value4});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items[1U], items.back()};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.Concat(state.front(), items[1U]), pb.AggrConcat(pb.AggrConcat(state.back(), pb.NewDataLiteral<NUdf::EDataSlot::String>(", ")), items.back())};
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.Unwrap(state.front(), landmine, __FILE__, __LINE__, 0), pb.AggrConcat(pb.AggrConcat(keys.front(), pb.NewDataLiteral<NUdf::EDataSlot::String>(": ")), state.back())};
+ }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.back(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key one: value 1, value 4, value 5, value 1, value 2");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key two: value 2, value 3, value 3, value 4");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestThinAllLambdas) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto tupleType = pb.NewTupleType({});
+ const auto data = pb.NewTuple({});
+
+ const auto list = pb.NewList(tupleType, {data, data, data, data});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(list),
+ [](TRuntimeNode) -> TRuntimeNode::TList { return {}; }),
+ [](TRuntimeNode::TList items) { return items; },
+ [](TRuntimeNode::TList, TRuntimeNode::TList items) { return items; },
+ [](TRuntimeNode::TList, TRuntimeNode::TList, TRuntimeNode::TList state) { return state; },
+ [](TRuntimeNode::TList, TRuntimeNode::TList state) { return state; }),
+ [&](TRuntimeNode::TList) { return pb.NewTuple({}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLWideLastCombinerPerfTest) {
+ Y_UNIT_TEST_LLVM(TestSumDoubleBooleanKeys) {
+ TSetup<LLVM> setup;
+
+ double positive = 0.0, negative = 0.0;
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ (sample.second > 0.0 ? positive : negative) += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.AggrGreater(items.front(), pb.NewDataLiteral(0.0))}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return items; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.front())}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { return state; }),
+ [&](TRuntimeNode::TList items) { return items.front(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); });
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto first = value.GetElement(0);
+ const auto second = value.GetElement(1);
+ const auto t2 = TInstant::Now();
+
+ if (first.template Get<double>() > 0.0) {
+ UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), positive);
+ UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), negative);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(first.template Get<double>(), negative);
+ UNIT_ASSERT_VALUES_EQUAL(second.template Get<double>(), positive);
+ }
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleBooleanKeys) {
+ TSetup<LLVM> setup;
+
+ double pSum = 0.0, nSum = 0.0, pMax = 0.0, nMax = -1000.0, pMin = 1000.0, nMin = 0.0;
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ if (sample.second > 0.0) {
+ pSum += sample.second;
+ pMax = std::max(pMax, sample.second);
+ pMin = std::min(pMin, sample.second);
+ } else {
+ nSum += sample.second;
+ nMax = std::max(nMax, sample.second);
+ nMin = std::min(nMin, sample.second);
+ }
+ }
+
+ const auto cppTime = TInstant::Now() - t;
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<double>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.AggrGreater(items.front(), pb.NewDataLiteral(0.0))}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front(), items.front(), items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.AggrAdd(state.front(), items.front()), pb.AggrMin(state[1U], items.front()), pb.AggrMax(state.back(), items.back()) };
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList state) -> TRuntimeNode::TList { return state; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), items, [](const std::pair<i8, double> s){ return ToValue<double>(s.second); });
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto first = value.GetElement(0);
+ const auto second = value.GetElement(1);
+ const auto t2 = TInstant::Now();
+
+ if (first.GetElement(0).template Get<double>() > 0.0) {
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), pSum);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), pMin);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), pMax);
+
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), nSum);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), nMin);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), nMax);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(0).template Get<double>(), nSum);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(1).template Get<double>(), nMin);
+ UNIT_ASSERT_VALUES_EQUAL(first.GetElement(2).template Get<double>(), nMax);
+
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(0).template Get<double>(), pSum);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(1).template Get<double>(), pMin);
+ UNIT_ASSERT_VALUES_EQUAL(second.GetElement(2).template Get<double>(), pMax);
+ }
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleSmallKey) {
+ TSetup<LLVM> setup;
+
+ std::unordered_map<i8, double> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ expects.emplace(sample.first, 0.0).first->second += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<i8, double>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back())}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), state.front()}; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ for (const auto& sample : I8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), ptr[i].GetElement(1).template Get<double>());
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<i8, double> l, const std::pair<i8, double> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleSmallKey) {
+ TSetup<LLVM> setup;
+
+ std::unordered_map<i8, std::array<double, 3U>> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : I8Samples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, std::numeric_limits<double>::max(), std::numeric_limits<double>::min()}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<i8, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<i8>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back())}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { state.insert(state.cbegin(), keys.front()); return state; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(I8Samples.size(), items));
+ for (const auto& sample : I8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElement(0).template Get<i8>(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<i8, std::array<double, 3U>> l, const std::pair<i8, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestSumDoubleStringKey) {
+ TSetup<LLVM> setup;
+
+ std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size());
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); });
+
+ std::unordered_map<std::string, double> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : stringI8Samples) {
+ expects.emplace(sample.first, 0.0).first->second += sample.second;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::string_view, double>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back())}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), state.front()}; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items));
+ for (const auto& sample : stringI8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElements()->AsStringRef(), ptr[i].GetElement(1).template Get<double>());
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, double> l, const std::pair<std::string_view, double> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumDoubleStringKey) {
+ TSetup<LLVM> setup;
+
+ std::vector<std::pair<std::string, double>> stringI8Samples(I8Samples.size());
+ std::transform(I8Samples.cbegin(), I8Samples.cend(), stringI8Samples.begin(), [](std::pair<i8, double> src){ return std::make_pair(ToString(src.first), src.second); });
+
+ std::unordered_map<std::string, std::array<double, 3U>> expects(201);
+ const auto t = TInstant::Now();
+ for (const auto& sample : stringI8Samples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::string_view, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewDataType(NUdf::TDataType<const char*>::Id), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(item, 0U), pb.Nth(item, 1U) }; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back())}; },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { state.insert(state.cbegin(), keys.front()); return state; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(stringI8Samples.size(), items));
+ for (const auto& sample : stringI8Samples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[0] = NUdf::TUnboxedValuePod::Embedded(sample.first);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ two.emplace_back(ptr[i].GetElements()->AsStringRef(), std::array<double, 3U>{ptr[i].GetElement(1).template Get<double>(), ptr[i].GetElement(2).template Get<double>(), ptr[i].GetElement(3).template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::string_view, std::array<double, 3U>> l, const std::pair<std::string_view, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestMinMaxSumTupleKey) {
+ TSetup<LLVM> setup;
+
+ std::vector<std::pair<std::pair<ui32, std::string>, double>> pairSamples(Ui16Samples.size());
+ std::transform(Ui16Samples.cbegin(), Ui16Samples.cend(), pairSamples.begin(), [](std::pair<ui16, double> src){ return std::make_pair(std::make_pair(ui32(src.first / 10U % 100U), ToString(src.first % 10U)), src.second); });
+
+ struct TPairHash { size_t operator()(const std::pair<ui32, std::string>& p) const { return CombineHashes(std::hash<ui32>()(p.first), std::hash<std::string_view>()(p.second)); } };
+
+ std::unordered_map<std::pair<ui32, std::string>, std::array<double, 3U>, TPairHash> expects;
+ const auto t = TInstant::Now();
+ for (const auto& sample : pairSamples) {
+ auto& item = expects.emplace(sample.first, std::array<double, 3U>{0.0, +1E7, -1E7}).first->second;
+ std::get<0U>(item) += sample.second;
+ std::get<1U>(item) = std::min(std::get<1U>(item), sample.second);
+ std::get<2U>(item) = std::max(std::get<2U>(item), sample.second);
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::pair<ui32, std::string>, std::array<double, 3U>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({pb.NewTupleType({pb.NewDataType(NUdf::TDataType<ui32>::Id), pb.NewDataType(NUdf::TDataType<const char*>::Id)}), pb.NewDataType(NUdf::TDataType<double>::Id)}));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return { pb.Nth(pb.Nth(item, 0U), 0U), pb.Nth(pb.Nth(item, 0U), 1U), pb.Nth(item, 1U) }; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.front(), items[1U]}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items.back(), items.back(), items.back()}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.AggrAdd(state.front(), items.back()), pb.AggrMin(state[1U], items.back()), pb.AggrMax(state.back(), items.back()) };
+ },
+ [&](TRuntimeNode::TList keys, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {keys.front(), keys.back(), state.front(), state[1U], state.back()}; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple({pb.NewTuple({items[0U], items[1U]}), items[2U], items[3U], items[4U]}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(pairSamples.size(), items));
+ for (const auto& sample : pairSamples) {
+ NUdf::TUnboxedValue* pair = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(2U, pair);
+ pair[1] = NUdf::TUnboxedValuePod(sample.second);
+ NUdf::TUnboxedValue* keys = nullptr;
+ pair[0] = graph->GetHolderFactory().CreateDirectArrayHolder(2U, keys);
+ keys[0] = NUdf::TUnboxedValuePod(sample.first.first);
+ keys[1] = NUdf::TUnboxedValuePod::Embedded(sample.first.second);
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ const auto elements = ptr[i].GetElements();
+ two.emplace_back(std::make_pair(elements[0].GetElement(0).template Get<ui32>(), (elements[0].GetElements()[1]).AsStringRef()), std::array<double, 3U>{elements[1].template Get<double>(), elements[2].template Get<double>(), elements[3].template Get<double>()});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> l, const std::pair<std::pair<ui32, std::string_view>, std::array<double, 3U>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+
+ Y_UNIT_TEST_LLVM(TestTpch) {
+ TSetup<LLVM> setup;
+
+ struct TPairHash { size_t operator()(const std::pair<std::string_view, std::string_view>& p) const { return CombineHashes(std::hash<std::string_view>()(p.first), std::hash<std::string_view>()(p.second)); } };
+
+ std::unordered_map<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>, TPairHash> expects;
+ const auto t = TInstant::Now();
+ for (auto& sample : TpchSamples) {
+ if (std::get<0U>(sample) <= border) {
+ const auto& ins = expects.emplace(std::pair<std::string_view, std::string_view>{std::get<1U>(sample), std::get<2U>(sample)}, std::pair<ui64, std::array<double, 5U>>{0ULL, {0., 0., 0., 0., 0.}});
+ auto& item = ins.first->second;
+ ++item.first;
+ std::get<0U>(item.second) += std::get<3U>(sample);
+ std::get<1U>(item.second) += std::get<5U>(sample);
+ std::get<2U>(item.second) += std::get<6U>(sample);
+ const auto v = std::get<3U>(sample) * (1. - std::get<5U>(sample));
+ std::get<3U>(item.second) += v;
+ std::get<4U>(item.second) += v * (1. + std::get<4U>(sample));
+ }
+ }
+ for (auto& item : expects) {
+ std::get<1U>(item.second.second) /= item.second.first;
+ }
+ const auto cppTime = TInstant::Now() - t;
+
+ std::vector<std::pair<std::pair<std::string, std::string>, std::pair<ui64, std::array<double, 5U>>>> one, two;
+ one.reserve(expects.size());
+ two.reserve(expects.size());
+
+ one.insert(one.cend(), expects.cbegin(), expects.cend());
+ std::sort(one.begin(), one.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; });
+
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto listType = pb.NewListType(pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui64>::Id),
+ pb.NewDataType(NUdf::TDataType<const char*>::Id),
+ pb.NewDataType(NUdf::TDataType<const char*>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id),
+ pb.NewDataType(NUdf::TDataType<double>::Id)
+ }));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideLastCombiner(
+ pb.WideFilter(pb.ExpandMap(pb.ToFlow(TRuntimeNode(list, false)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U), pb.Nth(item, 3U), pb.Nth(item, 4U), pb.Nth(item, 5U), pb.Nth(item, 6U)}; }),
+ [&](TRuntimeNode::TList items) { return pb.AggrLessOrEqual(items.front(), pb.NewDataLiteral<ui64>(border)); }
+ ),
+ [&](TRuntimeNode::TList item) -> TRuntimeNode::TList { return {item[1U], item[2U]}; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ const auto price = items[3U];
+ const auto disco = items[5U];
+ const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco));
+ return {pb.NewDataLiteral<ui64>(1ULL), price, disco, items[6U], v, pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), items[4U]))};
+ },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ const auto price = items[3U];
+ const auto disco = items[5U];
+ const auto v = pb.Mul(price, pb.Sub(pb.NewDataLiteral<double>(1.), disco));
+ return {pb.Increment(state[0U]), pb.AggrAdd(state[1U], price), pb.AggrAdd(state[2U], disco), pb.AggrAdd(state[3U], items[6U]), pb.AggrAdd(state[4U], v), pb.AggrAdd(state[5U], pb.Mul(v, pb.Add(pb.NewDataLiteral<double>(1.), items[4U])))};
+ },
+ [&](TRuntimeNode::TList key, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {key.front(), key.back(), state[0U], state[1U], pb.Div(state[2U], state[0U]), state[3U], state[4U], state[5U]}; }),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn, {list});
+ NUdf::TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(TpchSamples.size(), items));
+ for (const auto& sample : TpchSamples) {
+ NUdf::TUnboxedValue* elements = nullptr;
+ *items++ = graph->GetHolderFactory().CreateDirectArrayHolder(7U, elements);
+ elements[0] = NUdf::TUnboxedValuePod(std::get<0U>(sample));
+ elements[1] = NUdf::TUnboxedValuePod::Embedded(std::get<1U>(sample));
+ elements[2] = NUdf::TUnboxedValuePod::Embedded(std::get<2U>(sample));
+ elements[3] = NUdf::TUnboxedValuePod(std::get<3U>(sample));
+ elements[4] = NUdf::TUnboxedValuePod(std::get<4U>(sample));
+ elements[5] = NUdf::TUnboxedValuePod(std::get<5U>(sample));
+ elements[6] = NUdf::TUnboxedValuePod(std::get<6U>(sample));
+ }
+
+ const auto t1 = TInstant::Now();
+ const auto& value = graph->GetValue();
+ const auto t2 = TInstant::Now();
+
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), expects.size());
+
+ const auto ptr = value.GetElements();
+ for (size_t i = 0ULL; i < expects.size(); ++i) {
+ const auto elements = ptr[i].GetElements();
+ two.emplace_back(std::make_pair(elements[0].AsStringRef(), elements[1].AsStringRef()), std::pair<ui64, std::array<double, 5U>>{elements[2].template Get<ui64>(), {elements[3].template Get<double>(), elements[4].template Get<double>(), elements[5].template Get<double>(), elements[6].template Get<double>(), elements[7].template Get<double>()}});
+ }
+
+ std::sort(two.begin(), two.end(), [](const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> l, const std::pair<std::pair<std::string_view, std::string_view>, std::pair<ui64, std::array<double, 5U>>> r){ return l.first < r.first; });
+ UNIT_ASSERT_VALUES_EQUAL(one, two);
+
+ Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
+ }
+}
+#endif
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp
new file mode 100644
index 00000000000..f7fdc9e0d4b
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_condense_ut.cpp
@@ -0,0 +1,173 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u
+Y_UNIT_TEST_SUITE(TMiniKQLWideCondense1Test) {
+ Y_UNIT_TEST_LLVM(TestConcatItemsToKey) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCondense1(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items.front(), pb.AggrConcat(pb.AggrConcat(items.front(), pb.NewDataLiteral<NUdf::EDataSlot::String>(": ")), items.back())};
+ },
+ [&](TRuntimeNode::TList items, TRuntimeNode::TList state) {
+ return pb.AggrNotEquals(items.front(), state.front());
+ },
+ [&](TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {state.front(), pb.AggrConcat(pb.AggrConcat(state.back(), pb.NewDataLiteral<NUdf::EDataSlot::String>(", ")), items.back())};
+ }),
+ [&](TRuntimeNode::TList items) { return items.back(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key one: very long value 1");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "key two: very long value 2, very long value 3");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long key one: very long value 4");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "very long key two: very long value 5, very long value 6, very long value 7, very long value 8, very long value 9");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSwitchByBoolFieldAndDontUseKey) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto boolType = pb.NewDataType(NUdf::TDataType<bool>::Id);
+ const auto tupleType = pb.NewTupleType({pb.NewOptionalType(dataType), dataType, boolType});
+
+ const auto key0 = pb.NewEmptyOptional(pb.NewOptionalType(dataType));
+ const auto key1 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("one"));
+ const auto key2 = pb.NewOptional(pb.NewDataLiteral<NUdf::EDataSlot::String>("two"));
+
+ const auto trueVal = pb.NewDataLiteral<bool>(true);
+ const auto falseVal = pb.NewDataLiteral<bool>(false);
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {key0, value1, trueVal});
+ const auto data2 = pb.NewTuple(tupleType, {key1, value2, falseVal});
+ const auto data3 = pb.NewTuple(tupleType, {key2, value3, falseVal});
+ const auto data4 = pb.NewTuple(tupleType, {key0, value4, trueVal});
+ const auto data5 = pb.NewTuple(tupleType, {key1, value5, falseVal});
+ const auto data6 = pb.NewTuple(tupleType, {key2, value6, falseVal});
+ const auto data7 = pb.NewTuple(tupleType, {key0, value7, falseVal});
+ const auto data8 = pb.NewTuple(tupleType, {key1, value8, falseVal});
+ const auto data9 = pb.NewTuple(tupleType, {key2, value9, trueVal});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCondense1(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Unwrap(pb.Nth(item, 0U), landmine, __FILE__, __LINE__, 0), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList {
+ return {items[1U]};
+ },
+ [&](TRuntimeNode::TList items, TRuntimeNode::TList) {
+ return items.back();
+ },
+ [&](TRuntimeNode::TList items, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.AggrConcat(pb.AggrConcat(state.front(), pb.NewDataLiteral<NUdf::EDataSlot::String>("; ")), items[1U])};
+ }),
+ [&](TRuntimeNode::TList items) { return items.front(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 1; value 2; value 3");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 4; value 5; value 6; value 7; value 8");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "value 9");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestThinAllLambdas) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto tupleType = pb.NewTupleType({});
+
+ const auto data = pb.NewTuple({});
+
+ const auto list = pb.NewList(tupleType, {data, data, data, data});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideCondense1(pb.ExpandMap(pb.ToFlow(list),
+ [](TRuntimeNode) -> TRuntimeNode::TList { return {}; }),
+ [](TRuntimeNode::TList items) { return items; },
+ [&](TRuntimeNode::TList, TRuntimeNode::TList) { return pb.NewDataLiteral<bool>(true); },
+ [](TRuntimeNode::TList, TRuntimeNode::TList state) { return state;}),
+ [&](TRuntimeNode::TList) { return pb.NewTuple({}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+#endif
+}
+}
+
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp
new file mode 100644
index 00000000000..5d5a9e7aa73
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_filter_ut.cpp
@@ -0,0 +1,386 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u
+Y_UNIT_TEST_SUITE(TMiniKQLWideFilterTest) {
+ Y_UNIT_TEST_LLVM(TestPredicateExpression) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(
+ pb.NarrowMap(
+ pb.WideFilter(
+ pb.ExpandMap(
+ pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }
+ ),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ const auto v = pb.If(
+ pb.Exists(items.front()),
+ pb.NewOptional(pb.NewDataLiteral<bool>(true)),
+ pb.NewEmptyOptionalDataLiteral(NUdf::TDataType<bool>::Id)
+ );
+ return pb.Coalesce(v, pb.NewDataLiteral<bool>(false));
+ }
+ ),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ )
+ );
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ NUdf::TUnboxedValue value = graph->GetValue();
+
+ NUdf::TUnboxedValue v;
+ UNIT_ASSERT_VALUES_EQUAL(value.Fetch(v), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_VALUES_EQUAL(v.GetElement(0).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(v.GetElement(1).template Get<i32>(), -2);
+
+ UNIT_ASSERT_VALUES_EQUAL(value.Fetch(v), NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_VALUES_EQUAL(v.GetElement(0).template Get<i32>(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(v.GetElement(1).template Get<i32>(), -3);
+
+ UNIT_ASSERT_VALUES_EQUAL(value.Fetch(v), NUdf::EFetchStatus::Finish);
+ }
+
+ Y_UNIT_TEST_LLVM(TestCheckedFieldPasstrought) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items[1U]); }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestCheckedFieldUnusedAfter) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items[1U]); }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items.back(), items.front()}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -2);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDotCalculateUnusedField) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Unwrap(pb.Nth(item, 2U), landmine, __FILE__, __LINE__, 0)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items[1U]); }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestWithLimitCheckedFieldUsedAfter) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(9)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(7)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(6)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ pb.NewDataLiteral<ui64>(2ULL), [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items.front()); }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.AggrAdd(items.back(), items.front()); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 8);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestWithLimitCheckedFieldUnusedAfter) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ pb.NewDataLiteral<ui64>(2ULL), [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items.front()); }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.back(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<i32>(), -3);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestTakeWhile) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTakeWhile(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items.front()); }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -1);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestTakeWhileInclusive) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTakeWhileInclusive(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.Exists(items.front()); }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipWhile) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideSkipWhile(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.AggrGreater(items.back(), pb.NewOptional(pb.NewDataLiteral<i32>(-3))); }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 3);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipWhileInclusive) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideSkipWhileInclusive(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.AggrGreater(items.back(), pb.NewOptional(pb.NewDataLiteral<i32>(-3))); }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFilterByBooleanField) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, pb.NewDataType(NUdf::TDataType<bool>::Id), dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewDataLiteral(true), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewDataLiteral(false), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewDataLiteral(true), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewDataLiteral(false), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideFilter(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items[1U]; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items.back(), items.front()}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -3);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), +3);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+#endif
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp
new file mode 100644
index 00000000000..4add5f00ea2
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_map_ut.cpp
@@ -0,0 +1,253 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u
+Y_UNIT_TEST_SUITE(TMiniKQLWideMapTest) {
+ Y_UNIT_TEST_LLVM(TestSimpleSwap) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items[2U], items[1U], items[0U] }); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -1);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -3);
+ UNIT_ASSERT(!item.GetElement(1));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 3);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestThinLambda) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode) -> TRuntimeNode::TList { return {}; }),
+ [&](TRuntimeNode::TList items) { return items; }),
+ [&](TRuntimeNode::TList) -> TRuntimeNode { return pb.NewTuple({}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestWideMap) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.AggrMin(items[0], items[1]), pb.AggrMax(items[1], items[2]), pb.AggrAdd(items[0], items[2])}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -2);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -3);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 0);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 4);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDotCalculateUnusedField) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(0)), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-4))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto landmine = pb.NewDataLiteral<NUdf::EDataSlot::String>("ACHTUNG MINEN!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.Mul(items.front(), items.back()), pb.Unwrap(items[1], landmine, __FILE__, __LINE__, 0), pb.Add(items.front(), items.back())}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items.back(), items.front()}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -9);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), 0);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -16);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestPasstroughtFieldSplitAsIs) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(-5)), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-4)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-7)), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-4))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items[1U], pb.Mul(items.front(), items.back()), items[1U]}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -5);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -1);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -5);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -7);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -9);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), -7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), -16);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestFieldBothWayPasstroughtAndArg) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewOptionalType(pb.NewDataType(NUdf::TDataType<i32>::Id));
+ const auto tupleType = pb.NewTupleType({dataType, dataType, dataType});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(1)), pb.NewOptional(pb.NewDataLiteral<i32>(-5)), pb.NewOptional(pb.NewDataLiteral<i32>(-1))});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(2)), pb.NewOptional(pb.NewDataLiteral<i32>(-4)), pb.NewOptional(pb.NewDataLiteral<i32>(-2))});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(3)), pb.NewOptional(pb.NewDataLiteral<i32>(-7)), pb.NewOptional(pb.NewDataLiteral<i32>(-3))});
+ const auto data4 = pb.NewTuple(tupleType, {pb.NewOptional(pb.NewDataLiteral<i32>(4)), pb.NewEmptyOptional(dataType), pb.NewOptional(pb.NewDataLiteral<i32>(-4))});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideMap(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U), pb.Nth(item, 2U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {items[1U], pb.Sub(items.front(), items.back()), pb.Minus(items[1U])}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -5);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 5);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 4);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 4);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<i32>(), -7);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 6);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(2).template Get<i32>(), 7);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT(!item.GetElement(0));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<i32>(), 8);
+ UNIT_ASSERT(!item.GetElement(2));
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+#endif
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp
new file mode 100644
index 00000000000..87158beed65
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_nodes_ut.cpp
@@ -0,0 +1,124 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 18u
+Y_UNIT_TEST_SUITE(TMiniKQLWideNodesTest) {
+ // TDOD: fixme
+#if 0
+ Y_UNIT_TEST_LLVM(TestWideDiscard) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.Discard(pb.ExpandMap(pb.ToFlow(list), [](TRuntimeNode) { return TRuntimeNode::TList(); })));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+#endif
+
+ Y_UNIT_TEST_LLVM(TestDiscard) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto data0 = pb.NewDataLiteral<NUdf::EDataSlot::String>("000");
+ const auto data1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("100");
+ const auto data2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("200");
+ const auto data3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("300");
+ const auto dataType = pb.NewDataType(NUdf::TDataType<char*>::Id);
+ const auto list = pb.NewList(dataType, {data0, data1, data2, data3});
+
+ const auto pgmReturn = pb.FromFlow(pb.Discard(pb.ToFlow(list)));
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ UNIT_ASSERT_VALUES_EQUAL(NUdf::EFetchStatus::Finish, iterator.Fetch(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestTakeOverSource) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.Take(pb.Source(), pb.NewDataLiteral<ui64>(666ULL)), [&](TRuntimeNode::TList) { return pb.NewTuple({}); } ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ UNIT_ASSERT_VALUES_EQUAL(graph->GetValue().GetListLength(), 666ULL);
+ }
+
+ Y_UNIT_TEST_LLVM(TestSkipAndTake) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.ListFromRange(pb.NewDataLiteral<ui32>(100U), pb.NewDataLiteral<ui32>(666U), pb.NewDataLiteral<ui32>(3U));
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.Take(pb.Skip(pb.ExpandMap(pb.ToFlow(pb.Enumerate(list)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 1U), pb.Nth(item, 0U)}; }),
+ pb.NewDataLiteral<ui64>(42ULL)), pb.NewDataLiteral<ui64>(4ULL)),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple({items.back(), items.front()}); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 42);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 226);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 43);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 229);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 44);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 232);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(0).template Get<ui64>(), 45);
+ UNIT_ASSERT_VALUES_EQUAL(item.GetElement(1).template Get<ui32>(), 235);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TestDoNotCalculateSkipped) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto list = pb.ListFromRange(pb.NewDataLiteral<ui64>(100ULL), pb.NewDataLiteral<ui64>(135ULL), pb.NewDataLiteral<ui64>(5ULL));
+
+ const auto trap = pb.NewDataLiteral<NUdf::EDataSlot::String>("IT'S A TRAP!");
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.Skip(pb.WideMap(pb.ExpandMap(pb.ToFlow(pb.Enumerate(list)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 1U), pb.Nth(item, 0U)}; }),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode::TList { return {pb.Unwrap(pb.Div(items.front(), items.back()), trap, __FILE__, __LINE__, 0)}; }),
+ pb.NewDataLiteral<ui64>(3ULL)),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 38ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 30ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 25ULL);
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui64>(), 21ULL);
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+}
+#endif
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp
new file mode 100644
index 00000000000..093ae70d352
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_stream_ut.cpp
@@ -0,0 +1,58 @@
+#include "mkql_computation_node_ut.h"
+
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 36u
+Y_UNIT_TEST_SUITE(TMiniKQLWideStreamTest) {
+
+Y_UNIT_TEST_LLVM(TestSimple) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto ui64Type = pb.NewDataType(NUdf::TDataType<ui64>::Id);
+ const auto tupleType = pb.NewTupleType({ui64Type, ui64Type});
+
+ const auto data1 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(1), pb.NewDataLiteral<ui64>(10)});
+ const auto data2 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(2), pb.NewDataLiteral<ui64>(20)});
+ const auto data3 = pb.NewTuple(tupleType, {pb.NewDataLiteral<ui64>(3), pb.NewDataLiteral<ui64>(30)});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3});
+ const auto flow = pb.ToFlow(list);
+
+ const auto wideFlow = pb.ExpandMap(flow, [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ return {pb.Nth(item, 0U), pb.Nth(item, 1U)};
+ });
+
+ const auto wideStream = pb.FromFlow(wideFlow);
+ const auto newWideFlow = pb.ToFlow(wideStream);
+
+ const auto narrowFlow = pb.NarrowMap(newWideFlow, [&](TRuntimeNode::TList items) -> TRuntimeNode {
+ return pb.Sub(items[1], items[0]);
+ });
+ const auto pgmReturn = pb.Collect(narrowFlow);
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 9);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 18);
+
+ UNIT_ASSERT(iterator.Next(item));
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui64>(), 27);
+
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+}
+}
+
+#endif
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp
new file mode 100644
index 00000000000..b52b6175878
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_wide_top_sort_ut.cpp
@@ -0,0 +1,651 @@
+#include "mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/mkql_runtime_version.h>
+
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+#include <cstring>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 33u
+Y_UNIT_TEST_SUITE(TMiniKQLWideTopTest) {
+ Y_UNIT_TEST_LLVM(TopByFirstKeyAsc) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTop(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.NewDataLiteral<ui64>(4ULL), {{0U, pb.NewDataLiteral<bool>(true)}}),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 3");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 2");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TopByFirstKeyDesc) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTop(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.NewDataLiteral<ui64>(6ULL), {{0U, pb.NewDataLiteral<bool>(false)}}),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 7");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 5");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 6");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 8");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 9");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TopBySecondKeyAsc) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTop(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.NewDataLiteral<ui64>(3ULL), {{1U, pb.NewDataLiteral<bool>(true)}}),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 3");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 2");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TopBySecondKeyDesc) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTop(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.NewDataLiteral<ui64>(2ULL), {{1U, pb.NewDataLiteral<bool>(false)}}),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 8");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 9");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TopSortByFirstSecondAscDesc) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTopSort(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.NewDataLiteral<ui64>(4ULL), {{0U, pb.NewDataLiteral<bool>(true)}, {1U, pb.NewDataLiteral<bool>(false)}}),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 3");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 2");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TopSortByFirstSecondDescAsc) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTopSort(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.NewDataLiteral<ui64>(6ULL), {{0U, pb.NewDataLiteral<bool>(false)}, {1U, pb.NewDataLiteral<bool>(true)}}),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 5");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 6");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 7");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 8");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 9");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TopSortBySecondFirstAscDesc) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTopSort(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.NewDataLiteral<ui64>(4ULL), {{1U, pb.NewDataLiteral<bool>(true)}, {0U, pb.NewDataLiteral<bool>(false)}}),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 2");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 3");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TopSortBySecondFirstDescAsc) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTopSort(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ pb.NewDataLiteral<ui64>(6ULL), {{1U, pb.NewDataLiteral<bool>(false)}, {0U, pb.NewDataLiteral<bool>(true)}}),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 9");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 8");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 7");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 6");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 5");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+
+ Y_UNIT_TEST_LLVM(TopSortLargeList) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto minusday = i64(-24LL * 60LL * 60LL * 1000000LL); // -1 Day
+ const auto step = pb.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&minusday, sizeof(minusday)));
+ const auto list = pb.ListFromRange(pb.NewTzDataLiteral<NUdf::TTzDate>(30000u, 42u), pb.NewTzDataLiteral<NUdf::TTzDate>(10000u, 42u), step);
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideTopSort(pb.ExpandMap(pb.ToFlow(pb.Enumerate(list)),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList {
+ const auto utf = pb.ToString<true>(pb.Nth(item, 1U));
+ const auto day = pb.StrictFromString(pb.Substring(utf, pb.NewDataLiteral<ui32>(8U), pb.NewDataLiteral<ui32>(2U)), pb.NewDataType(NUdf::EDataSlot::Uint8));
+ return {pb.Nth(item, 0U), utf, day, pb.Nth(item, 1U)};
+ }),
+ pb.NewDataLiteral<ui64>(7ULL), {{2U, pb.NewDataLiteral<bool>(true)}, {0U, pb.NewDataLiteral<bool>(false)}}),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items[1]; }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "1997-06-01,Africa/Mbabane");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "1997-07-01,Africa/Mbabane");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "1997-08-01,Africa/Mbabane");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "1997-09-01,Africa/Mbabane");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "1997-10-01,Africa/Mbabane");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "1997-11-01,Africa/Mbabane");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item, "1997-12-01,Africa/Mbabane");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+#endif
+
+#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 34u
+Y_UNIT_TEST_SUITE(TMiniKQLWideSortTest) {
+ Y_UNIT_TEST_LLVM(SortByFirstKeyAsc) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ const auto dataType = pb.NewDataType(NUdf::TDataType<const char*>::Id);
+ const auto tupleType = pb.NewTupleType({dataType, dataType});
+
+ const auto keyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("key one");
+ const auto keyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("key two");
+
+ const auto longKeyOne = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key one");
+ const auto longKeyTwo = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long key two");
+
+ const auto value1 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 1");
+ const auto value2 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 2");
+ const auto value3 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 3");
+ const auto value4 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 4");
+ const auto value5 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 5");
+ const auto value6 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 6");
+ const auto value7 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 7");
+ const auto value8 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 8");
+ const auto value9 = pb.NewDataLiteral<NUdf::EDataSlot::String>("very long value 9");
+
+ const auto data1 = pb.NewTuple(tupleType, {keyOne, value1});
+
+ const auto data2 = pb.NewTuple(tupleType, {keyTwo, value2});
+ const auto data3 = pb.NewTuple(tupleType, {keyTwo, value3});
+
+ const auto data4 = pb.NewTuple(tupleType, {longKeyOne, value4});
+
+ const auto data5 = pb.NewTuple(tupleType, {longKeyTwo, value5});
+ const auto data6 = pb.NewTuple(tupleType, {longKeyTwo, value6});
+ const auto data7 = pb.NewTuple(tupleType, {longKeyTwo, value7});
+ const auto data8 = pb.NewTuple(tupleType, {longKeyTwo, value8});
+ const auto data9 = pb.NewTuple(tupleType, {longKeyTwo, value9});
+
+ const auto list = pb.NewList(tupleType, {data1, data2, data3, data4, data5, data6, data7, data8, data9});
+
+ const auto pgmReturn = pb.Collect(pb.NarrowMap(pb.WideSort(pb.ExpandMap(pb.ToFlow(list),
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0U), pb.Nth(item, 1U)}; }),
+ {{0U, pb.NewDataLiteral<bool>(true)}}),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return pb.NewTuple(tupleType, items); }
+ ));
+
+ const auto graph = setup.BuildGraph(pgmReturn);
+ const auto iterator = graph->GetValue().GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 1");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 3");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 2");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key one");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 4");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 9");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 8");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 7");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 6");
+ UNIT_ASSERT(iterator.Next(item));
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(0), "very long key two");
+ UNBOXED_VALUE_STR_EQUAL(item.GetElement(1), "very long value 5");
+ UNIT_ASSERT(!iterator.Next(item));
+ UNIT_ASSERT(!iterator.Next(item));
+ }
+}
+#endif
+
+}
+}
+
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/run_codegen.cmd b/ydb/library/yql/minikql/comp_nodes/ut/run_codegen.cmd
new file mode 100644
index 00000000000..6dd1238df85
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/run_codegen.cmd
@@ -0,0 +1,9 @@
+@echo off
+call ya make -o ..\..\..\..\ && yql-minikql-comp_nodes-ut.exe
+if errorlevel 1 goto :fail
+echo --- DONE ---
+pause
+exit /b
+:fail
+echo --- FAIL ---
+pause
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/ya.make b/ydb/library/yql/minikql/comp_nodes/ut/ya.make
new file mode 100644
index 00000000000..2303397956f
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ut/ya.make
@@ -0,0 +1,83 @@
+UNITTEST_FOR(ydb/library/yql/minikql/comp_nodes/llvm)
+
+FORK_SUBTESTS()
+
+SPLIT_FACTOR(60)
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ TIMEOUT(3600)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCS(
+ mkql_test_factory.cpp
+ mkql_bit_utils_ut.cpp
+ mkql_block_compress_ut.cpp
+ mkql_block_skiptake_ut.cpp
+ mkql_blocks_ut.cpp
+ mkql_combine_ut.cpp
+ mkql_condense_ut.cpp
+ mkql_decimal_ut.cpp
+ mkql_chain_map_ut.cpp
+ mkql_chopper_ut.cpp
+ mkql_filters_ut.cpp
+ mkql_flatmap_ut.cpp
+ mkql_fromstring_ut.cpp
+ mkql_multihopping_saveload_ut.cpp
+ mkql_multihopping_ut.cpp
+ mkql_multimap_ut.cpp
+ mkql_fold_ut.cpp
+ mkql_heap_ut.cpp
+ mkql_compare_ut.cpp
+ mkql_computation_node_ut.cpp
+ mkql_group_ut.cpp
+ mkql_dict_ut.cpp
+ mkql_join_ut.cpp
+ mkql_join_dict_ut.cpp
+ mkql_grace_join_ut.cpp
+ mkql_map_join_ut.cpp
+ mkql_safe_circular_buffer_ut.cpp
+ mkql_sort_ut.cpp
+ mkql_switch_ut.cpp
+ mkql_todict_ut.cpp
+ mkql_variant_ut.cpp
+ mkql_wide_chain_map_ut.cpp
+ mkql_wide_chopper_ut.cpp
+ mkql_wide_combine_ut.cpp
+ mkql_wide_condense_ut.cpp
+ mkql_wide_filter_ut.cpp
+ mkql_wide_map_ut.cpp
+ mkql_wide_nodes_ut.cpp
+ mkql_wide_stream_ut.cpp
+ mkql_wide_top_sort_ut.cpp
+ mkql_listfromrange_ut.cpp
+ mkql_mapnext_ut.cpp
+ mkql_rh_hash_ut.cpp
+)
+
+PEERDIR(
+ ydb/library/yql/public/udf
+ ydb/library/yql/public/udf/arrow
+ ydb/library/yql/public/udf/service/exception_policy
+ ydb/library/yql/sql/pg_dummy
+)
+
+CFLAGS(
+ -mprfchw
+)
+
+YQL_LAST_ABI_VERSION()
+
+IF (MKQL_RUNTIME_VERSION)
+ CFLAGS(
+ -DMKQL_RUNTIME_VERSION=$MKQL_RUNTIME_VERSION
+ )
+ENDIF()
+
+REQUIREMENTS(ram:10)
+
+END()
diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make
new file mode 100644
index 00000000000..424ecfb86d0
--- /dev/null
+++ b/ydb/library/yql/minikql/comp_nodes/ya.make
@@ -0,0 +1,8 @@
+RECURSE(
+ llvm
+ no_llvm
+)
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/minikql/computation/CMakeLists.txt b/ydb/library/yql/minikql/computation/CMakeLists.txt
index e519707fa41..4716de1ef37 100644
--- a/ydb/library/yql/minikql/computation/CMakeLists.txt
+++ b/ydb/library/yql/minikql/computation/CMakeLists.txt
@@ -7,3 +7,5 @@
add_subdirectory(llvm)
+add_subdirectory(no_llvm)
+add_subdirectory(ut)
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp
new file mode 100644
index 00000000000..098363250a5
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp
@@ -0,0 +1,124 @@
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <vector>
+#include <utility>
+#include <algorithm>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+struct TSetup {
+ TSetup(TScopedAlloc& alloc)
+ : Alloc(alloc)
+ {
+ FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ RandomProvider = CreateDeterministicRandomProvider(1);
+ TimeProvider = CreateDeterministicTimeProvider(10000000);
+
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
+ THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
+ Explorer.Walk(pgm.GetNode(), *Env);
+ TComputationPatternOpts opts(Alloc.Ref(), *Env, GetBuiltinFactory(),
+ FunctionRegistry.Get(),
+ NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, "OFF", EGraphPerProcess::Multi);
+ Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
+ TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
+ return Pattern->Clone(compOpts);
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+
+ TScopedAlloc& Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
+ IComputationPattern::TPtr Pattern;
+};
+}
+
+Y_UNIT_TEST_SUITE(TestCompactMultiDict) {
+ Y_UNIT_TEST(TestIterate) {
+ TScopedAlloc alloc(__LOCATION__);
+
+ TSetup setup(alloc);
+
+ const std::vector<std::pair<ui32, std::vector<ui32>>> items = {{1, {1, 2}}, {2, {1}}, {3, {0}}, {6, {1, 7}}};
+
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+ TVector<TRuntimeNode> rItems;
+ for (auto& [k, vv]: items) {
+ for (auto& v: vv) {
+ rItems.push_back(pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<ui32>(k), pgmBuilder.NewDataLiteral<ui32>(v)}));
+ }
+ }
+ auto ui32Type = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto list = pgmBuilder.NewList(pgmBuilder.NewTupleType({ui32Type, ui32Type}), rItems);
+
+ auto dict = pgmBuilder.ToHashedDict(list, /*all*/true,
+ [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.Nth(item, 0); },
+ [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.Nth(item, 1); },
+ /*isCompact*/true,
+ items.size());
+
+ auto graph = setup.BuildGraph(dict, {});
+ NUdf::TUnboxedValue res = graph->GetValue();
+
+ std::vector<ui32> keyVals;
+ for (NUdf::TUnboxedValue keys = res.GetKeysIterator(), v; keys.Next(v);) {
+ keyVals.push_back(v.Get<ui32>());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(keyVals.size(), items.size());
+ std::sort(keyVals.begin(), keyVals.end());
+ UNIT_ASSERT(
+ std::equal(keyVals.begin(), keyVals.end(), items.begin(),
+ [](ui32 l, const std::pair<ui32, std::vector<ui32>>& r) { return l == r.first; }
+ )
+ );
+
+ std::vector<std::vector<ui32>> origPayloads;
+ for (auto& [k, vv]: items) {
+ origPayloads.push_back(vv);
+ std::sort(origPayloads.back().begin(), origPayloads.back().end());
+ }
+ std::sort(origPayloads.begin(), origPayloads.end());
+
+ std::vector<std::vector<ui32>> payloadVals;
+ for (NUdf::TUnboxedValue payloads = res.GetPayloadsIterator(), v; payloads.Next(v);) {
+ payloadVals.emplace_back();
+ for (NUdf::TUnboxedValue i = v.GetListIterator(), p; i.Next(p);) {
+ payloadVals.back().push_back(p.Get<ui32>());
+ }
+ std::sort(payloadVals.back().begin(), payloadVals.back().end());
+ }
+ std::sort(payloadVals.begin(), payloadVals.end());
+ UNIT_ASSERT_VALUES_EQUAL(origPayloads, payloadVals);
+
+ std::vector<std::pair<ui32, std::vector<ui32>>> vals;
+ for (NUdf::TUnboxedValue values = res.GetDictIterator(), k, payloads; values.NextPair(k, payloads);) {
+ vals.emplace_back(k.Get<ui32>(), std::vector<ui32>{});
+ for (NUdf::TUnboxedValue i = payloads.GetListIterator(), p; i.Next(p);) {
+ vals.back().second.push_back(p.Get<ui32>());
+ }
+ std::sort(vals.back().second.begin(), vals.back().second.end());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(items, vals);
+ }
+}
+}
+}
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
index 373976d79f4..4aa1892f907 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
@@ -144,6 +144,8 @@ public:
{
#ifndef NDEBUG
AllocState.ActiveMemInfo.emplace(MemInfo.Get(), MemInfo);
+#else
+ Y_UNUSED(AllocState);
#endif
}
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp
new file mode 100644
index 00000000000..5453f8b5909
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp
@@ -0,0 +1,380 @@
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/mkql_node_cast.h>
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+ TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
+ return CreateDeterministicRandomProvider(1);
+ }
+
+ TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
+ return CreateDeterministicTimeProvider(10000000);
+ }
+
+ TComputationNodeFactory GetAuxCallableFactory() {
+ return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+ if (callable.GetType()->GetName() == "OneYieldStream") {
+ return new TExternalComputationNode(ctx.Mutables);
+ }
+
+ return GetBuiltinFactory()(callable, ctx);
+ };
+ }
+
+ struct TSetup {
+ TSetup(TScopedAlloc& alloc)
+ : Alloc(alloc)
+ {
+ FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ RandomProvider = CreateRandomProvider();
+ TimeProvider = CreateTimeProvider();
+
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
+ THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
+ Explorer.Walk(pgm.GetNode(), *Env);
+ TComputationPatternOpts opts(Alloc.Ref(), *Env, GetAuxCallableFactory(),
+ FunctionRegistry.Get(),
+ NUdf::EValidateMode::None, NUdf::EValidatePolicy::Fail, "OFF", EGraphPerProcess::Multi);
+ Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
+ TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
+ return Pattern->Clone(compOpts);
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+
+ TScopedAlloc& Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
+ IComputationPattern::TPtr Pattern;
+ };
+
+ struct TStreamWithYield : public NUdf::TBoxedValue {
+ TStreamWithYield(const TUnboxedValueVector& items, ui32 yieldPos, ui32 index)
+ : Items(items)
+ , YieldPos(yieldPos)
+ , Index(index)
+ {}
+
+ private:
+ TUnboxedValueVector Items;
+ ui32 YieldPos;
+ ui32 Index;
+
+ ui32 GetTraverseCount() const override {
+ return 0;
+ }
+
+ NUdf::TUnboxedValue Save() const override {
+ return NUdf::TUnboxedValue::Zero();
+ }
+
+ void Load(const NUdf::TStringRef& state) override {
+ Y_UNUSED(state);
+ }
+
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) final {
+ if (Index >= Items.size()) {
+ return NUdf::EFetchStatus::Finish;
+ }
+ if (Index == YieldPos) {
+ return NUdf::EFetchStatus::Yield;
+ }
+ result = Items[Index++];
+ return NUdf::EFetchStatus::Ok;
+ }
+ };
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLSaveLoadTest) {
+ Y_UNIT_TEST(TestSqueezeSaveLoad) {
+ TScopedAlloc alloc(__LOCATION__);
+
+ const std::vector<ui32> items = {2, 3, 4, 5, 6, 7, 8};
+
+ auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto streamType = pgmBuilder.NewStreamType(dataType);
+
+ TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType);
+ auto streamNode = inStream.Build();
+
+ auto pgmReturn = pgmBuilder.Squeeze(
+ TRuntimeNode(streamNode, false),
+ pgmBuilder.NewDataLiteral<ui32>(1),
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pgmBuilder.Add(item, state);
+ },
+ [](TRuntimeNode state) {
+ return state;
+ },
+ [](TRuntimeNode state) {
+ return state;
+ });
+
+ TUnboxedValueVector streamItems;
+ for (auto item : items) {
+ streamItems.push_back(NUdf::TUnboxedValuePod(item));
+ }
+
+ auto graph = setup.BuildGraph(pgmReturn, {streamNode});
+ auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
+ return graph;
+ };
+
+ for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) {
+ TSetup setup1(alloc);
+ auto graph1 = buildGraph(setup1, yieldPos, 0);
+
+ auto root1 = graph1->GetValue();
+ NUdf::TUnboxedValue res;
+ auto status = root1.Fetch(res);
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
+
+ TString graphState;
+ SaveGraphState(&root1, 1, 0ULL, graphState);
+
+ TSetup setup2(alloc);
+ auto graph2 = buildGraph(setup2, -1, yieldPos);
+
+ auto root2 = graph2->GetValue();
+ LoadGraphState(&root2, 1, 0ULL, graphState);
+
+ status = root2.Fetch(res);
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36);
+
+ status = root2.Fetch(res);
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
+ }
+ }
+
+ Y_UNIT_TEST(TestSqueeze1SaveLoad) {
+ TScopedAlloc alloc(__LOCATION__);
+
+ const std::vector<ui32> items = {1, 2, 3, 4, 5, 6, 7, 8};
+
+ auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto dataType = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto streamType = pgmBuilder.NewStreamType(dataType);
+
+ TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", streamType);
+ auto streamNode = inStream.Build();
+
+ auto pgmReturn = pgmBuilder.Squeeze1(
+ TRuntimeNode(streamNode, false),
+ [](TRuntimeNode item) {
+ return item;
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) {
+ return pgmBuilder.Add(item, state);
+ },
+ [](TRuntimeNode state) {
+ return state;
+ },
+ [](TRuntimeNode state) {
+ return state;
+ });
+
+ TUnboxedValueVector streamItems;
+ for (auto item : items) {
+ streamItems.push_back(NUdf::TUnboxedValuePod(item));
+ }
+
+ auto graph = setup.BuildGraph(pgmReturn, {streamNode});
+ auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
+ return graph;
+ };
+
+ for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) {
+ TSetup setup1(alloc);
+ auto graph1 = buildGraph(setup1, yieldPos, 0);
+
+ auto root1 = graph1->GetValue();
+
+ NUdf::TUnboxedValue res;
+ auto status = root1.Fetch(res);
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
+
+ TString graphState;
+ SaveGraphState(&root1, 1, 0ULL, graphState);
+
+ TSetup setup2(alloc);
+ auto graph2 = buildGraph(setup2, -1, yieldPos);
+
+ auto root2 = graph2->GetValue();
+ LoadGraphState(&root2, 1, 0ULL, graphState);
+
+ status = root2.Fetch(res);
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Ok);
+ UNIT_ASSERT_EQUAL(res.Get<ui32>(), 36);
+
+ status = root2.Fetch(res);
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
+ }
+ }
+
+ Y_UNIT_TEST(TestHoppingSaveLoad) {
+ TScopedAlloc alloc(__LOCATION__);
+
+ const std::vector<std::pair<i64, ui32>> items = {
+ {1, 2},
+ {2, 3},
+ {15, 4},
+ {23, 6},
+ {24, 5},
+ {25, 7},
+ {40, 2},
+ {47, 1},
+ {51, 6},
+ {59, 2},
+ {85, 8},
+ {55, 1000},
+ {200, 0}
+ };
+
+ auto buildGraph = [&items] (TSetup& setup, ui32 yieldPos, ui32 startIndex) -> THolder<IComputationGraph> {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto structType = pgmBuilder.NewEmptyStructType();
+ structType = pgmBuilder.NewStructType(structType, "time",
+ pgmBuilder.NewDataType(NUdf::TDataType<NUdf::TTimestamp>::Id));
+ structType = pgmBuilder.NewStructType(structType, "sum",
+ pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ auto timeIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("time");
+ auto sumIndex = AS_TYPE(TStructType, structType)->GetMemberIndex("sum");
+
+ auto inStreamType = pgmBuilder.NewStreamType(structType);
+
+ TCallableBuilder inStream(pgmBuilder.GetTypeEnvironment(), "OneYieldStream", inStreamType);
+ auto streamNode = inStream.Build();
+
+ ui64 hop = 10, interval = 30, delay = 20;
+
+ auto pgmReturn = pgmBuilder.HoppingCore(
+ TRuntimeNode(streamNode, false),
+ [&](TRuntimeNode item) { // timeExtractor
+ return pgmBuilder.Member(item, "time");
+ },
+ [&](TRuntimeNode item) { // init
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", pgmBuilder.Member(item, "sum"));
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode item, TRuntimeNode state) { // update
+ auto add = pgmBuilder.AggrAdd(
+ pgmBuilder.Member(item, "sum"),
+ pgmBuilder.Member(state, "sum"));
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", add);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state) { // save
+ return pgmBuilder.Member(state, "sum");
+ },
+ [&](TRuntimeNode savedState) { // load
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", savedState);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state1, TRuntimeNode state2) { // merge
+ auto add = pgmBuilder.AggrAdd(
+ pgmBuilder.Member(state1, "sum"),
+ pgmBuilder.Member(state2, "sum"));
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", add);
+ return pgmBuilder.NewStruct(members);
+ },
+ [&](TRuntimeNode state, TRuntimeNode time) { // finish
+ Y_UNUSED(time);
+ std::vector<std::pair<std::string_view, TRuntimeNode>> members;
+ members.emplace_back("sum", pgmBuilder.Member(state, "sum"));
+ return pgmBuilder.NewStruct(members);
+ },
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&hop, sizeof(hop))), // hop
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&interval, sizeof(interval))), // interval
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::Interval>(NUdf::TStringRef((const char*)&delay, sizeof(delay))) // delay
+ );
+
+ auto graph = setup.BuildGraph(pgmReturn, {streamNode});
+
+ TUnboxedValueVector streamItems;
+ for (size_t i = 0; i < items.size(); ++i) {
+ NUdf::TUnboxedValue* itemsPtr;
+ auto structValues = graph->GetHolderFactory().CreateDirectArrayHolder(2, itemsPtr);
+ itemsPtr[timeIndex] = NUdf::TUnboxedValuePod(items[i].first);
+ itemsPtr[sumIndex] = NUdf::TUnboxedValuePod(items[i].second);
+ streamItems.push_back(std::move(structValues));
+ }
+
+ auto streamValue = NUdf::TUnboxedValuePod(new TStreamWithYield(streamItems, yieldPos, startIndex));
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), std::move(streamValue));
+ return graph;
+ };
+
+ for (ui32 yieldPos = 0; yieldPos < items.size(); ++yieldPos) {
+ std::vector<ui32> result;
+
+ TSetup setup1(alloc);
+ auto graph1 = buildGraph(setup1, yieldPos, 0);
+ auto root1 = graph1->GetValue();
+
+ NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
+ while (status == NUdf::EFetchStatus::Ok) {
+ NUdf::TUnboxedValue val;
+ status = root1.Fetch(val);
+ if (status == NUdf::EFetchStatus::Ok) {
+ result.push_back(val.GetElement(0).Get<ui32>());
+ }
+ }
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Yield);
+
+ TString graphState;
+ SaveGraphState(&root1, 1, 0ULL, graphState);
+
+ TSetup setup2(alloc);
+ auto graph2 = buildGraph(setup2, -1, yieldPos);
+ auto root2 = graph2->GetValue();
+ LoadGraphState(&root2, 1, 0ULL, graphState);
+
+ status = NUdf::EFetchStatus::Ok;
+ while (status == NUdf::EFetchStatus::Ok) {
+ NUdf::TUnboxedValue val;
+ status = root2.Fetch(val);
+ if (status == NUdf::EFetchStatus::Ok) {
+ result.push_back(val.GetElement(0).Get<ui32>());
+ }
+ }
+ UNIT_ASSERT_EQUAL(status, NUdf::EFetchStatus::Finish);
+
+ const std::vector<ui32> resultCompare = {5, 9, 27, 22, 21, 11, 11, 8, 8, 8, 8};
+ UNIT_ASSERT_EQUAL(result, resultCompare);
+ }
+ }
+}
+
+} // namespace NMiniKQL
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp
new file mode 100644
index 00000000000..a07f1b05c7f
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp
@@ -0,0 +1,126 @@
+#include "mkql_computation_node_list.h"
+
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+Y_UNIT_TEST_SUITE(TestListRepresentation) {
+ Y_UNIT_TEST(Test) {
+ TMemoryUsageInfo memInfo(TStringBuf("test"));
+ TScopedAlloc alloc(__LOCATION__);
+ using TListType = TListRepresentation<ui32, 256>;
+ TListType list1;
+ auto it1 = list1.GetIterator();
+ UNIT_ASSERT(it1.AtEnd());
+
+ TListType list2(list1, 0);
+ auto it2 = list2.GetIterator();
+ UNIT_ASSERT(!it2.AtEnd());
+ UNIT_ASSERT_VALUES_EQUAL(it2.Current(), 0);
+ it2.Next();
+ UNIT_ASSERT(it2.AtEnd());
+
+ TVector<TListType> lists;
+ lists.push_back(list1);
+ const ui32 n = 100;
+ for (ui32 i = 0; i < n; ++i) {
+ auto c = i;
+ lists.push_back(TListType(lists.back(), std::move(c)));
+ }
+
+ for (ui32 i = 0; i < n; ++i) {
+ auto list = lists[i];
+ auto it = list.GetIterator();
+ ui32 expected = 0;
+ while (!it.AtEnd()) {
+ UNIT_ASSERT_VALUES_EQUAL(it.Current(), expected);
+ ++expected;
+ it.Next();
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(expected, i);
+ }
+
+ for (ui32 i = 0; i < n; ++i) {
+ auto list = lists[i];
+ auto it = list.GetReverseIterator();
+ ui32 expected = i;
+ while (!it.AtEnd()) {
+ UNIT_ASSERT_VALUES_EQUAL(it.Current(), --expected);
+ it.Next();
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(expected, 0);
+ }
+
+ TVector<TListType> rlists;
+ rlists.push_back(list1);
+ for (ui32 i = 0; i < n; ++i) {
+ auto c = i;
+ rlists.push_back(TListType(std::move(c), rlists.back()));
+ }
+
+ for (ui32 i = 0; i < n; ++i) {
+ auto list = rlists[i];
+ auto it = list.GetIterator();
+ ui32 expected = i;
+ while (!it.AtEnd()) {
+ UNIT_ASSERT_VALUES_EQUAL(it.Current(), --expected);
+ it.Next();
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(expected, 0);
+ }
+
+ for (ui32 i = 0; i < n; ++i) {
+ auto list = rlists[i];
+ auto it = list.GetReverseIterator();
+ ui32 expected = 0;
+ while (!it.AtEnd()) {
+ UNIT_ASSERT_VALUES_EQUAL(it.Current(), expected);
+ ++expected;
+ it.Next();
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(expected, i);
+ }
+
+ TVector<TListType> zlists;
+ zlists.push_back(list1);
+ for (ui32 i = 0; i < n; ++i) {
+ zlists.push_back(zlists.back().Append(2 * i));
+ zlists.push_back(zlists.back().Prepend(2 * i + 1));
+ }
+
+ for (ui32 i = 0; i < 2 * n; ++i) {
+ ui32 k = (i + 1) / 2;
+ auto list = zlists[i];
+ auto it = list.GetIterator();
+ ui32 expected = 2 * k + 1;
+ if ((i % 2))
+ expected -= 2;
+
+ if (i >= 2) {
+ while (!it.AtEnd()) {
+ expected -= 2;
+ UNIT_ASSERT_VALUES_EQUAL(it.Current(), expected);
+ it.Next();
+ if (expected == 1)
+ break;
+ }
+ }
+
+ expected = 0;
+ while (!it.AtEnd()) {
+ UNIT_ASSERT_VALUES_EQUAL(it.Current(), expected);
+ expected += 2;
+ it.Next();
+ }
+
+ UNIT_ASSERT_VALUES_EQUAL(expected, 2 * k);
+ }
+ }
+}
+}
+}
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp
new file mode 100644
index 00000000000..6234ef827bb
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp
@@ -0,0 +1,863 @@
+#include "mkql_computation_node_pack.h"
+#include "mkql_computation_node_holders.h"
+#include "mkql_block_builder.h"
+#include "mkql_block_reader.h"
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/public/udf/arrow/util.h>
+
+#include <library/cpp/random_provider/random_provider.h>
+#include <ydb/library/yql/minikql/aligned_page_pool.h>
+
+#include <ydb/library/yql/public/udf/udf_value.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+#include <util/generic/ylimits.h>
+#include <util/generic/xrange.h>
+#include <util/generic/maybe.h>
+#include <util/string/cast.h>
+#include <util/string/builder.h>
+#include <util/system/hp_timer.h>
+#include <util/system/sanitizers.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+#ifdef WITH_VALGRIND
+constexpr static size_t PERFORMANCE_COUNT = 0x1000;
+#elifdef NDEBUG
+constexpr static size_t PERFORMANCE_COUNT = NSan::PlainOrUnderSanitizer(0x4000000, 0x1000);
+#else
+constexpr static size_t PERFORMANCE_COUNT = NSan::PlainOrUnderSanitizer(0x1000000, 0x1000);
+#endif
+
+template<bool Fast, bool Transport>
+struct TPackerTraits;
+
+template<bool Fast>
+struct TPackerTraits<Fast, false> {
+ using TPackerType = TValuePackerGeneric<Fast>;
+};
+
+template<bool Fast>
+struct TPackerTraits<Fast, true> {
+ using TPackerType = TValuePackerTransport<Fast>;
+};
+
+using NDetails::TChunkedInputBuffer;
+
+template<bool Fast, bool Transport>
+class TMiniKQLComputationNodePackTest: public TTestBase {
+ using TValuePackerType = typename TPackerTraits<Fast, Transport>::TPackerType;
+protected:
+ TMiniKQLComputationNodePackTest()
+ : FunctionRegistry(CreateFunctionRegistry(CreateBuiltinRegistry()))
+ , RandomProvider(CreateDefaultRandomProvider())
+ , Alloc(__LOCATION__)
+ , Env(Alloc)
+ , PgmBuilder(Env, *FunctionRegistry)
+ , MemInfo("Memory")
+ , HolderFactory(Alloc.Ref(), MemInfo, FunctionRegistry.Get())
+ , ArrowPool_(arrow::default_memory_pool())
+ {
+ }
+
+ void TestNumericTypes() {
+ TestNumericType<bool>(NUdf::TDataType<bool>::Id);
+ TestNumericType<ui8>(NUdf::TDataType<ui8>::Id);
+ TestNumericType<i32>(NUdf::TDataType<i32>::Id);
+ TestNumericType<i64>(NUdf::TDataType<i64>::Id);
+ TestNumericType<ui32>(NUdf::TDataType<ui32>::Id);
+ TestNumericType<ui64>(NUdf::TDataType<ui64>::Id);
+ TestNumericType<float>(NUdf::TDataType<float>::Id);
+ TestNumericType<double>(NUdf::TDataType<double>::Id);
+ }
+
+ void TestOptionalNumericTypes() {
+ TestOptionalNumericType<bool>(NUdf::TDataType<bool>::Id);
+ TestOptionalNumericType<ui8>(NUdf::TDataType<ui8>::Id);
+ TestOptionalNumericType<i32>(NUdf::TDataType<i32>::Id);
+ TestOptionalNumericType<i64>(NUdf::TDataType<i64>::Id);
+ TestOptionalNumericType<ui32>(NUdf::TDataType<ui32>::Id);
+ TestOptionalNumericType<ui64>(NUdf::TDataType<ui64>::Id);
+ TestOptionalNumericType<float>(NUdf::TDataType<float>::Id);
+ TestOptionalNumericType<double>(NUdf::TDataType<double>::Id);
+ }
+
+ void TestStringTypes() {
+ TestStringType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ TestStringType(NUdf::TDataType<char*>::Id);
+ TestStringType(NUdf::TDataType<NUdf::TYson>::Id);
+ TestStringType(NUdf::TDataType<NUdf::TJson>::Id);
+ }
+
+ void TestOptionalStringTypes() {
+ TestOptionalStringType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ TestOptionalStringType(NUdf::TDataType<char*>::Id);
+ TestOptionalStringType(NUdf::TDataType<NUdf::TYson>::Id);
+ TestOptionalStringType(NUdf::TDataType<NUdf::TJson>::Id);
+ }
+
+ void TestListType() {
+ TDefaultListRepresentation listValues;
+ for (ui32 val: xrange(0, 10)) {
+ listValues = listValues.Append(NUdf::TUnboxedValuePod(val));
+ }
+ TType* listType = PgmBuilder.NewListType(PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ const NUdf::TUnboxedValue value = HolderFactory.CreateDirectListHolder(std::move(listValues));
+ const auto uValue = TestPackUnpack(listType, value, "Type:List(ui32)");
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetListLength(), 10);
+ ui32 i = 0;
+ const auto iter = uValue.GetListIterator();
+ for (NUdf::TUnboxedValue item; iter.Next(item); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), i);
+ }
+ UNIT_ASSERT_VALUES_EQUAL(i, 10);
+ }
+
+ void TestListOfOptionalsType() {
+ TDefaultListRepresentation listValues;
+ for (ui32 i: xrange(0, 10)) {
+ NUdf::TUnboxedValue uVal = NUdf::TUnboxedValuePod();
+ if (i % 2) {
+ uVal = MakeString(TString(i * 2, '0' + i));
+ }
+ listValues = listValues.Append(std::move(uVal));
+ }
+ TType* listType = PgmBuilder.NewListType(PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)));
+ const NUdf::TUnboxedValue value = HolderFactory.CreateDirectListHolder(std::move(listValues));
+ const auto uValue = TestPackUnpack(listType, value, "Type:List(Optional(utf8))");
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetListLength(), 10);
+ ui32 i = 0;
+ const auto iter = uValue.GetListIterator();
+ for (NUdf::TUnboxedValue uVal; iter.Next(uVal); ++i) {
+ if (i % 2) {
+ UNIT_ASSERT(uVal);
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(uVal.AsStringRef()), TString(i * 2, '0' + i));
+ } else {
+ UNIT_ASSERT(!uVal);
+ }
+ }
+ UNIT_ASSERT_VALUES_EQUAL(i, 10);
+ }
+
+ void TestTupleType() {
+ std::vector<TType*> tupleElemenTypes;
+ tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
+ tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
+ TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
+
+ TUnboxedValueVector tupleElemens;
+ tupleElemens.push_back(MakeString("01234567890123456789"));
+ tupleElemens.push_back(MakeString("01234567890"));
+ tupleElemens.push_back(NUdf::TUnboxedValuePod());
+ tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
+ tupleElemens.push_back(NUdf::TUnboxedValuePod());
+ tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
+
+ const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(tupleElemens);
+ const auto uValue = TestPackUnpack(tupleType, value, "Type:Tuple");
+
+ {
+ auto e = uValue.GetElement(0);
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789");
+ }
+ {
+ auto e = uValue.GetElement(1);
+ UNIT_ASSERT(e);
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890");
+ }
+ {
+ auto e = uValue.GetElement(2);
+ UNIT_ASSERT(!e);
+ }
+ {
+ auto e = uValue.GetElement(3);
+ UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
+ }
+ {
+ auto e = uValue.GetElement(4);
+ UNIT_ASSERT(!e);
+ }
+ {
+ auto e = uValue.GetElement(5);
+ UNIT_ASSERT(e);
+ UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
+ }
+ }
+
+ void TestStructType() {
+ const std::vector<std::pair<std::string_view, TType*>> structElemenTypes = {
+ {"a", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)},
+ {"b", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)},
+ {"c", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)},
+ {"d", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)},
+ {"e", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id, true)},
+ {"f", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id, true)}
+ };
+ TType* structType = PgmBuilder.NewStructType(structElemenTypes);
+
+ TUnboxedValueVector structElemens;
+ structElemens.push_back(MakeString("01234567890123456789"));
+ structElemens.push_back(MakeString("01234567890"));
+ structElemens.push_back(NUdf::TUnboxedValuePod());
+ structElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
+ structElemens.push_back(NUdf::TUnboxedValuePod());
+ structElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
+
+ const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(structElemens);
+ const auto uValue = TestPackUnpack(structType, value, "Type:Struct");
+
+ {
+ auto e = uValue.GetElement(0);
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789");
+ }
+ {
+ auto e = uValue.GetElement(1);
+ UNIT_ASSERT(e);
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890");
+ }
+ {
+ auto e = uValue.GetElement(2);
+ UNIT_ASSERT(!e);
+ }
+ {
+ auto e = uValue.GetElement(3);
+ UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
+ }
+ {
+ auto e = uValue.GetElement(4);
+ UNIT_ASSERT(!e);
+ }
+ {
+ auto e = uValue.GetElement(5);
+ UNIT_ASSERT(e);
+ UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
+ }
+ }
+
+ void TestOptionalType() {
+ TType* type = PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id);
+ type = PgmBuilder.NewOptionalType(type);
+ type = PgmBuilder.NewOptionalType(type);
+ type = PgmBuilder.NewOptionalType(type);
+
+ NUdf::TUnboxedValue uValue = NUdf::TUnboxedValuePod();
+ uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:null");
+ UNIT_ASSERT(!uValue);
+
+ uValue = NUdf::TUnboxedValuePod(ui64(123));
+ uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(opt(opt(123)))");
+ UNIT_ASSERT_VALUES_EQUAL(uValue.Get<ui64>(), 123);
+
+ uValue = NUdf::TUnboxedValuePod().MakeOptional().MakeOptional();
+ uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(opt(null))");
+ UNIT_ASSERT(uValue);
+ uValue = uValue.GetOptionalValue();
+ UNIT_ASSERT(uValue);
+ uValue = uValue.GetOptionalValue();
+ UNIT_ASSERT(!uValue);
+
+ uValue = NUdf::TUnboxedValuePod().MakeOptional();
+ uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(null)");
+ UNIT_ASSERT(uValue);
+ uValue = uValue.GetOptionalValue();
+ UNIT_ASSERT(!uValue);
+ }
+
+ void TestDictType() {
+ TType* keyType = PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
+ TType* payloadType = PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ TType* dictType = PgmBuilder.NewDictType(keyType, payloadType, false);
+
+ TValuesDictHashSingleFixedMap<ui32> map;
+ map[4] = NUdf::TUnboxedValuePod::Embedded("4");
+ map[10] = NUdf::TUnboxedValuePod::Embedded("10");
+ map[1] = NUdf::TUnboxedValuePod::Embedded("1");
+ const NUdf::TUnboxedValue value = HolderFactory.CreateDirectHashedSingleFixedMapHolder<ui32, false>(std::move(map), std::nullopt);
+
+ const auto uValue = TestPackUnpack(dictType, value, "Type:Dict");
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetDictLength(), 3);
+ const auto it = uValue.GetDictIterator();
+ for (NUdf::TUnboxedValue key, payload; it.NextPair(key, payload);) {
+ UNIT_ASSERT_VALUES_EQUAL(ToString(key.template Get<ui32>()), std::string_view(payload.AsStringRef()));
+ }
+ }
+
+ void TestVariantTypeOverStruct() {
+ const std::vector<std::pair<std::string_view, TType*>> structElemenTypes = {
+ {"a", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)},
+ {"b", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)},
+ {"d", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}
+ };
+ TType* structType = PgmBuilder.NewStructType(structElemenTypes);
+ TestVariantTypeImpl(PgmBuilder.NewVariantType(structType));
+ }
+
+ void TestVariantTypeOverTuple() {
+ std::vector<TType*> tupleElemenTypes;
+ tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
+ tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id));
+ TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
+ TestVariantTypeImpl(PgmBuilder.NewVariantType(tupleType));
+ }
+
+ void ValidateEmbeddedLength(TRope buf, const TString& info) {
+ size_t size = buf.GetSize();
+ TChunkedInputBuffer chunked(std::move(buf));
+ return ValidateEmbeddedLength(chunked, size, info);
+ }
+
+ void ValidateEmbeddedLength(TStringBuf buf, const TString& info) {
+ TChunkedInputBuffer chunked(buf);
+ return ValidateEmbeddedLength(chunked, buf.size(), info);
+ }
+
+ void ValidateEmbeddedLength(TChunkedInputBuffer& buf, size_t totalSize, const TString& info) {
+ if constexpr (!Fast) {
+ if (totalSize > 8) {
+ ui32 len = NDetails::GetRawData<ui32>(buf);
+ UNIT_ASSERT_VALUES_EQUAL_C(len + 4, totalSize, info);
+ } else {
+ ui32 len = NDetails::GetRawData<ui8>(buf);
+ UNIT_ASSERT_VALUES_EQUAL_C(((len & 0x0f) >> 1) + 1, totalSize, info);
+ }
+ }
+ }
+
+ void TestPackPerformance(TType* type, const NUdf::TUnboxedValuePod& uValue)
+ {
+ TValuePackerType packer(false, type);
+ const THPTimer timer;
+ for (size_t i = 0U; i < PERFORMANCE_COUNT; ++i)
+ packer.Pack(uValue);
+ Cerr << timer.Passed() << Endl;
+ }
+
+ NUdf::TUnboxedValue TestPackUnpack(TValuePackerType& packer, const NUdf::TUnboxedValuePod& uValue,
+ const TString& additionalMsg, const std::optional<ui32>& expectedLength = {})
+ {
+ auto packedValue = packer.Pack(uValue);
+ if constexpr (Transport) {
+ if (expectedLength) {
+ UNIT_ASSERT_VALUES_EQUAL_C(packedValue.size(), *expectedLength, additionalMsg);
+ }
+ ValidateEmbeddedLength(packedValue, additionalMsg);
+ return packer.Unpack(std::move(packedValue), HolderFactory);
+ } else {
+ if (expectedLength) {
+ UNIT_ASSERT_VALUES_EQUAL_C(packedValue.Size(), *expectedLength, additionalMsg);
+ }
+ ValidateEmbeddedLength(packedValue, additionalMsg);
+ return packer.Unpack(packedValue, HolderFactory);
+ }
+ }
+
+ NUdf::TUnboxedValue TestPackUnpack(TType* type, const NUdf::TUnboxedValuePod& uValue, const TString& additionalMsg,
+ const std::optional<ui32>& expectedLength = {})
+ {
+ TValuePackerType packer(false, type);
+ return TestPackUnpack(packer, uValue, additionalMsg, expectedLength);
+ }
+
+ template <typename T>
+ void TestNumericValue(T value, TValuePackerType& packer, const TString& typeDesc) {
+ TString additionalMsg = TStringBuilder() << typeDesc << ", Value:" << value;
+ auto uValue = TestPackUnpack(packer, NUdf::TUnboxedValuePod(value), additionalMsg);
+ UNIT_ASSERT_VALUES_EQUAL_C(uValue.template Get<T>(), value, additionalMsg);
+ }
+
+ template <typename T>
+ void TestNumericType(NUdf::TDataTypeId schemeType) {
+ TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
+ TValuePackerType packer(false, PgmBuilder.NewDataType(schemeType));
+
+ TestNumericValue<T>(Max<T>(), packer, typeDesc);
+ TestNumericValue<T>(Min<T>(), packer, typeDesc);
+ TestNumericValue<T>(T(0), packer, typeDesc);
+ TestNumericValue<T>(T(1), packer, typeDesc);
+ }
+
+ template <typename T>
+ void TestOptionalNumericValue(std::optional<T> value, TValuePackerType& packer, const TString& typeDesc,
+ const std::optional<ui32>& expectedLength = {})
+ {
+ TString additionalMsg = TStringBuilder() << typeDesc << "), Value:" << (value ? ToString(*value) : TString("null"));
+ const auto v = value ? NUdf::TUnboxedValuePod(*value) : NUdf::TUnboxedValuePod();
+ const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength);
+ if (value) {
+ UNIT_ASSERT_VALUES_EQUAL_C(uValue.template Get<T>(), *value, additionalMsg);
+ } else {
+ UNIT_ASSERT_C(!uValue, additionalMsg);
+ }
+ }
+
+ template <typename T>
+ void TestOptionalNumericType(NUdf::TDataTypeId schemeType) {
+ TString typeDesc = TStringBuilder() << ", Type:Optional(" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
+ TValuePackerType packer(false, PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(schemeType)));
+ TestOptionalNumericValue<T>(std::optional<T>(Max<T>()), packer, typeDesc);
+ TestOptionalNumericValue<T>(std::optional<T>(Min<T>()), packer, typeDesc);
+ TestOptionalNumericValue<T>(std::optional<T>(), packer, typeDesc, 1);
+ TestOptionalNumericValue<T>(std::optional<T>(0), packer, typeDesc);
+ TestOptionalNumericValue<T>(std::optional<T>(1), packer, typeDesc);
+ }
+
+ void TestStringValue(const std::string_view& value, TValuePackerType& packer, const TString& typeDesc, ui32 expectedLength) {
+ TString additionalMsg = TStringBuilder() << typeDesc << ", Value:" << value;
+ const auto v = NUdf::TUnboxedValue(MakeString(value));
+ const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength);
+ UNIT_ASSERT_VALUES_EQUAL_C(std::string_view(uValue.AsStringRef()), value, additionalMsg);
+ }
+
+ void TestStringType(NUdf::TDataTypeId schemeType) {
+ TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
+ TValuePackerType packer(false, PgmBuilder.NewDataType(schemeType));
+ TestStringValue("0123456789012345678901234567890123456789", packer, typeDesc, 40 + 4);
+ TestStringValue("[]", packer, typeDesc, Fast ? (2 + 4) : (2 + 1));
+ TestStringValue("1234567", packer, typeDesc, Fast ? (7 + 4) : (7 + 1));
+ TestStringValue("", packer, typeDesc, Fast ? (0 + 4) : (0 + 1));
+ TestStringValue("12345678", packer, typeDesc, 8 + 4);
+
+ TString hugeString(12345678, 'X');
+ TestStringValue(hugeString, packer, typeDesc, hugeString.size() + 4);
+ }
+
+ void TestUuidType() {
+ auto schemeType = NUdf::TDataType<NUdf::TUuid>::Id;
+ TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
+ TValuePackerType packer(false, PgmBuilder.NewDataType(schemeType));
+ TestStringValue("0123456789abcdef", packer, typeDesc, Fast ? 16 : (16 + 4));
+ }
+
+ void TestOptionalStringValue(std::optional<std::string_view> value, TValuePackerType& packer, const TString& typeDesc, ui32 expectedLength) {
+ TString additionalMsg = TStringBuilder() << typeDesc << "), Value:" << (value ? *value : TString("null"));
+ const auto v = value ? NUdf::TUnboxedValue(MakeString(*value)) : NUdf::TUnboxedValue();
+ const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength);
+ if (value) {
+ UNIT_ASSERT_VALUES_EQUAL_C(std::string_view(uValue.AsStringRef()), *value, additionalMsg);
+ } else {
+ UNIT_ASSERT_C(!uValue, additionalMsg);
+ }
+ }
+
+ void TestOptionalStringType(NUdf::TDataTypeId schemeType) {
+ TString typeDesc = TStringBuilder() << ", Type:Optional(" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
+ TValuePackerType packer(false, PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(schemeType)));
+ TestOptionalStringValue("0123456789012345678901234567890123456789", packer, typeDesc, Fast ? (40 + 4 + 1) : (40 + 4));
+ TestOptionalStringValue(std::nullopt, packer, typeDesc, 1);
+ TestOptionalStringValue("[]", packer, typeDesc, Fast ? (2 + 4 + 1) : (2 + 1));
+ TestOptionalStringValue("1234567", packer, typeDesc, Fast ? (7 + 4 + 1) : (7 + 1));
+ TestOptionalStringValue("", packer, typeDesc, Fast ? (0 + 4 + 1) : 1);
+ TestOptionalStringValue("12345678", packer, typeDesc, Fast ? (8 + 4 + 1) : (8 + 4));
+ }
+
+ void TestVariantTypeImpl(TType* variantType) {
+ TString descr = TStringBuilder() << "Type:Variant("
+ << static_cast<TVariantType*>(variantType)->GetUnderlyingType()->GetKindAsStr() << ')';
+ {
+ const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(MakeString("01234567890123456789"), 0);
+ const auto uValue = TestPackUnpack(variantType, value, descr);
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 0);
+ auto e = uValue.GetVariantItem();
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789");
+ }
+ {
+ const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(), 1);
+ const auto uValue = TestPackUnpack(variantType, value, descr);
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 1);
+ auto e = uValue.GetVariantItem();
+ UNIT_ASSERT(!e);
+ }
+ {
+ const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(ui64(12345)), 2);
+ const auto uValue = TestPackUnpack(variantType, value, descr);
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 2);
+ auto e = uValue.GetVariantItem();
+ UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345ull);
+ }
+ }
+
+ NUdf::TUnboxedValue MakeTupleValue(TType*& tupleType) {
+ std::vector<TType*> tupleElemenTypes;
+ tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
+ tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
+ tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
+
+ TUnboxedValueVector tupleElemens;
+ tupleElemens.push_back(MakeString("01234567890123456789"));
+ tupleElemens.push_back(MakeString("01234567890"));
+ tupleElemens.push_back(NUdf::TUnboxedValuePod());
+ tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
+ tupleElemens.push_back(NUdf::TUnboxedValuePod());
+ tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
+
+ return HolderFactory.VectorAsArray(tupleElemens);
+ }
+
+ void ValidateTupleValue(const NUdf::TUnboxedValue& value) {
+ using NYql::NUdf::TStringValue;
+ UNIT_ASSERT(value.IsBoxed());
+
+ auto e0 = value.GetElement(0);
+ auto e1 = value.GetElement(1);
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(e0.AsStringRef()), "01234567890123456789");
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(e1.AsStringRef()), "01234567890");
+ UNIT_ASSERT(!value.GetElement(2).HasValue());
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(3).Get<ui64>(), 12345);
+ UNIT_ASSERT(!value.GetElement(4).HasValue());
+ UNIT_ASSERT_VALUES_EQUAL(value.GetElement(5).Get<ui64>(), 12345);
+ }
+
+ void TestTuplePackPerformance() {
+ TType* tupleType;
+ const auto value = MakeTupleValue(tupleType);
+ TestPackPerformance(tupleType, value);
+ }
+
+ void TestPairPackPerformance() {
+ std::vector<TType*> tupleElemenTypes;
+ tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
+ TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
+
+ TUnboxedValueVector tupleElemens;
+ tupleElemens.push_back(NUdf::TUnboxedValuePod(ui32(12345)));
+ tupleElemens.push_back(NUdf::TUnboxedValuePod(ui32(67890)));
+
+ const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(tupleElemens);
+ TestPackPerformance(tupleType, value);
+ }
+
+ void TestShortStringPackPerformance() {
+ const auto v = NUdf::TUnboxedValuePod::Embedded("01234");
+ TType* type = PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ TestPackPerformance(type, v);
+ }
+
+ void TestIntegerPackPerformance() {
+ const auto& v = NUdf::TUnboxedValuePod(ui64("123456789ULL"));
+ TType* type = PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id);
+ TestPackPerformance(type, v);
+ }
+
+ void TestRopeSplit() {
+ if constexpr (Transport) {
+ TType* tupleType;
+ const auto value = MakeTupleValue(tupleType);
+
+ TValuePackerType packer(false, tupleType);
+
+ auto buffer = packer.Pack(value);
+
+ TString packed = buffer.ConvertToString();
+
+ if constexpr (Fast) {
+ UNIT_ASSERT_VALUES_EQUAL(packed.size(), 59);
+ } else {
+ UNIT_ASSERT_VALUES_EQUAL(packed.size(), 43);
+ }
+
+ for (size_t chunk = 1; chunk < packed.size(); ++chunk) {
+ TString first = packed.substr(0, chunk);
+ TString second = packed.substr(chunk);
+
+ TRope result(std::move(first));
+ result.Insert(result.End(), TRope(std::move(second)));
+
+ UNIT_ASSERT_VALUES_EQUAL(result.size(), packed.size());
+ UNIT_ASSERT(!result.IsContiguous());
+
+ ValidateTupleValue(packer.Unpack(std::move(result), HolderFactory));
+ }
+ }
+ }
+
+ void TestIncrementalPacking() {
+ if constexpr (Transport) {
+ auto itemType = PgmBuilder.NewDataType(NUdf::TDataType<char *>::Id);
+ auto listType = PgmBuilder.NewListType(itemType);
+ TValuePackerType packer(false, itemType);
+ TValuePackerType listPacker(false, listType);
+
+ TStringBuf str = "01234567890ABCDEFG";
+
+ size_t count = 500000;
+
+ for (size_t i = 0; i < count; ++i) {
+ NUdf::TUnboxedValue item(MakeString(str));
+ packer.AddItem(item);
+ }
+
+ auto serialized = packer.Finish();
+
+ auto listObj = listPacker.Unpack(TRope(serialized), HolderFactory);
+ UNIT_ASSERT_VALUES_EQUAL(listObj.GetListLength(), count);
+ ui32 i = 0;
+ const auto iter = listObj.GetListIterator();
+ for (NUdf::TUnboxedValue uVal; iter.Next(uVal); ++i) {
+ UNIT_ASSERT(uVal);
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(uVal.AsStringRef()), str);
+ }
+
+ TUnboxedValueBatch items;
+ packer.UnpackBatch(std::move(serialized), HolderFactory, items);
+ UNIT_ASSERT_VALUES_EQUAL(items.RowCount(), count);
+ items.ForEachRow([&](const NUdf::TUnboxedValue& value) {
+ UNIT_ASSERT(value);
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(value.AsStringRef()), str);
+ });
+ }
+ }
+
+ void DoTestBlockPacking(ui64 offset, ui64 len) {
+ if constexpr (Transport) {
+ auto strType = PgmBuilder.NewDataType(NUdf::TDataType<char*>::Id);
+ auto ui32Type = PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto ui64Type = PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id);
+ auto optStrType = PgmBuilder.NewOptionalType(strType);
+ auto optUi32Type = PgmBuilder.NewOptionalType(ui32Type);
+
+ auto tupleOptUi32StrType = PgmBuilder.NewTupleType({ optUi32Type, strType });
+ auto optTupleOptUi32StrType = PgmBuilder.NewOptionalType(tupleOptUi32StrType);
+
+ auto blockUi32Type = PgmBuilder.NewBlockType(ui32Type, TBlockType::EShape::Many);
+ auto blockOptStrType = PgmBuilder.NewBlockType(optStrType, TBlockType::EShape::Many);
+ auto scalarOptStrType = PgmBuilder.NewBlockType(optStrType, TBlockType::EShape::Scalar);
+ auto blockOptTupleOptUi32StrType = PgmBuilder.NewBlockType(optTupleOptUi32StrType, TBlockType::EShape::Many);
+ auto scalarUi64Type = PgmBuilder.NewBlockType(ui64Type, TBlockType::EShape::Scalar);
+
+ auto rowType = PgmBuilder.NewMultiType({ blockUi32Type, blockOptStrType, scalarOptStrType, blockOptTupleOptUi32StrType, scalarUi64Type });
+
+
+ ui64 blockLen = 1000;
+ UNIT_ASSERT_LE(offset + len, blockLen);
+
+ auto builder1 = MakeArrayBuilder(TTypeInfoHelper(), ui32Type, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(ui32Type)), nullptr);
+ auto builder2 = MakeArrayBuilder(TTypeInfoHelper(), optStrType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(optStrType)), nullptr);
+ auto builder3 = MakeArrayBuilder(TTypeInfoHelper(), optTupleOptUi32StrType, *ArrowPool_, CalcBlockLen(CalcMaxBlockItemSize(optTupleOptUi32StrType)), nullptr);
+
+ for (ui32 i = 0; i < blockLen; ++i) {
+ TBlockItem b1(i);
+ builder1->Add(b1);
+
+ TString a = "a string " + ToString(i);
+ TBlockItem b2 = (i % 2) ? TBlockItem(a) : TBlockItem();
+ builder2->Add(b2);
+
+ TBlockItem b3items[] = { (i % 2) ? TBlockItem(i) : TBlockItem(), TBlockItem(a) };
+ TBlockItem b3 = (i % 7) ? TBlockItem(b3items) : TBlockItem();
+ builder3->Add(b3);
+ }
+
+ std::string_view testScalarString = "foobar";
+ auto strbuf = std::make_shared<arrow::Buffer>((const ui8*)testScalarString.data(), testScalarString.size());
+
+ TVector<arrow::Datum> datums;
+ datums.emplace_back(builder1->Build(true));
+ datums.emplace_back(builder2->Build(true));
+ datums.emplace_back(arrow::Datum(std::make_shared<arrow::BinaryScalar>(strbuf)));
+ datums.emplace_back(builder3->Build(true));
+ datums.emplace_back(arrow::Datum(std::make_shared<arrow::UInt64Scalar>(blockLen)));
+
+ if (offset != 0 || len != blockLen) {
+ for (auto& datum : datums) {
+ if (datum.is_array()) {
+ datum = NYql::NUdf::DeepSlice(datum.array(), offset, len);
+ }
+ }
+ }
+ TUnboxedValueVector columns;
+ for (auto& datum : datums) {
+ columns.emplace_back(HolderFactory.CreateArrowBlock(std::move(datum)));
+ }
+
+ TValuePackerType packer(false, rowType, ArrowPool_);
+ packer.AddWideItem(columns.data(), columns.size());
+ TRope packed = packer.Finish();
+
+ TUnboxedValueBatch unpacked(rowType);
+ packer.UnpackBatch(std::move(packed), HolderFactory, unpacked);
+
+ UNIT_ASSERT_VALUES_EQUAL(unpacked.RowCount(), 1);
+
+ TUnboxedValueVector unpackedColumns;
+ unpacked.ForEachRowWide([&](const NYql::NUdf::TUnboxedValue* values, ui32 count) {
+ unpackedColumns.insert(unpackedColumns.end(), values, values + count);
+ });
+
+ UNIT_ASSERT_VALUES_EQUAL(unpackedColumns.size(), columns.size());
+ UNIT_ASSERT_VALUES_EQUAL(TArrowBlock::From(unpackedColumns.back()).GetDatum().scalar_as<arrow::UInt64Scalar>().value, blockLen);
+ UNIT_ASSERT_VALUES_EQUAL(TArrowBlock::From(unpackedColumns[2]).GetDatum().scalar_as<arrow::BinaryScalar>().value->ToString(), testScalarString);
+
+
+ auto reader1 = MakeBlockReader(TTypeInfoHelper(), ui32Type);
+ auto reader2 = MakeBlockReader(TTypeInfoHelper(), optStrType);
+ auto reader3 = MakeBlockReader(TTypeInfoHelper(), optTupleOptUi32StrType);
+
+ for (ui32 i = offset; i < len; ++i) {
+ TBlockItem b1 = reader1->GetItem(*TArrowBlock::From(unpackedColumns[0]).GetDatum().array(), i - offset);
+ UNIT_ASSERT_VALUES_EQUAL(b1.As<ui32>(), i);
+
+ TString a = "a string " + ToString(i);
+ TBlockItem b2 = reader2->GetItem(*TArrowBlock::From(unpackedColumns[1]).GetDatum().array(), i - offset);
+ if (i % 2) {
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(b2.AsStringRef()), a);
+ } else {
+ UNIT_ASSERT(!b2);
+ }
+
+ TBlockItem b3 = reader3->GetItem(*TArrowBlock::From(unpackedColumns[3]).GetDatum().array(), i - offset);
+ if (i % 7) {
+ auto elements = b3.GetElements();
+ if (i % 2) {
+ UNIT_ASSERT_VALUES_EQUAL(elements[0].As<ui32>(), i);
+ } else {
+ UNIT_ASSERT(!elements[0]);
+ }
+ UNIT_ASSERT_VALUES_EQUAL(std::string_view(elements[1].AsStringRef()), a);
+ } else {
+ UNIT_ASSERT(!b3);
+ }
+ }
+ }
+ }
+
+ void TestBlockPacking() {
+ DoTestBlockPacking(0, 1000);
+ }
+
+ void TestBlockPackingSliced() {
+ DoTestBlockPacking(19, 623);
+ }
+private:
+ TIntrusivePtr<NMiniKQL::IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TScopedAlloc Alloc;
+ TTypeEnvironment Env;
+ TProgramBuilder PgmBuilder;
+ TMemoryUsageInfo MemInfo;
+ THolderFactory HolderFactory;
+ arrow::MemoryPool* const ArrowPool_;
+};
+
+class TMiniKQLComputationNodeGenericPackTest: public TMiniKQLComputationNodePackTest<false, false> {
+ UNIT_TEST_SUITE(TMiniKQLComputationNodeGenericPackTest);
+ UNIT_TEST(TestNumericTypes);
+ UNIT_TEST(TestOptionalNumericTypes);
+ UNIT_TEST(TestStringTypes);
+ UNIT_TEST(TestUuidType);
+ UNIT_TEST(TestOptionalStringTypes);
+ UNIT_TEST(TestListType);
+ UNIT_TEST(TestListOfOptionalsType);
+ UNIT_TEST(TestTupleType);
+ UNIT_TEST(TestStructType);
+ UNIT_TEST(TestOptionalType);
+ UNIT_TEST(TestDictType);
+ UNIT_TEST(TestVariantTypeOverStruct);
+ UNIT_TEST(TestVariantTypeOverTuple);
+ UNIT_TEST(TestIntegerPackPerformance);
+ UNIT_TEST(TestShortStringPackPerformance);
+ UNIT_TEST(TestPairPackPerformance);
+ UNIT_TEST(TestTuplePackPerformance);
+ UNIT_TEST_SUITE_END();
+};
+
+class TMiniKQLComputationNodeGenericFastPackTest: public TMiniKQLComputationNodePackTest<true, false> {
+ UNIT_TEST_SUITE(TMiniKQLComputationNodeGenericFastPackTest);
+ UNIT_TEST(TestNumericTypes);
+ UNIT_TEST(TestOptionalNumericTypes);
+ UNIT_TEST(TestStringTypes);
+ UNIT_TEST(TestUuidType);
+ UNIT_TEST(TestOptionalStringTypes);
+ UNIT_TEST(TestListType);
+ UNIT_TEST(TestListOfOptionalsType);
+ UNIT_TEST(TestTupleType);
+ UNIT_TEST(TestStructType);
+ UNIT_TEST(TestOptionalType);
+ UNIT_TEST(TestDictType);
+ UNIT_TEST(TestVariantTypeOverStruct);
+ UNIT_TEST(TestVariantTypeOverTuple);
+ UNIT_TEST(TestIntegerPackPerformance);
+ UNIT_TEST(TestShortStringPackPerformance);
+ UNIT_TEST(TestPairPackPerformance);
+ UNIT_TEST(TestTuplePackPerformance);
+ UNIT_TEST_SUITE_END();
+};
+
+class TMiniKQLComputationNodeTransportPackTest: public TMiniKQLComputationNodePackTest<false, true> {
+ UNIT_TEST_SUITE(TMiniKQLComputationNodeTransportPackTest);
+ UNIT_TEST(TestNumericTypes);
+ UNIT_TEST(TestOptionalNumericTypes);
+ UNIT_TEST(TestStringTypes);
+ UNIT_TEST(TestUuidType);
+ UNIT_TEST(TestOptionalStringTypes);
+ UNIT_TEST(TestListType);
+ UNIT_TEST(TestListOfOptionalsType);
+ UNIT_TEST(TestTupleType);
+ UNIT_TEST(TestStructType);
+ UNIT_TEST(TestOptionalType);
+ UNIT_TEST(TestDictType);
+ UNIT_TEST(TestVariantTypeOverStruct);
+ UNIT_TEST(TestVariantTypeOverTuple);
+ UNIT_TEST(TestIntegerPackPerformance);
+ UNIT_TEST(TestShortStringPackPerformance);
+ UNIT_TEST(TestPairPackPerformance);
+ UNIT_TEST(TestTuplePackPerformance);
+ UNIT_TEST(TestRopeSplit);
+ UNIT_TEST(TestIncrementalPacking);
+ UNIT_TEST(TestBlockPacking);
+ UNIT_TEST(TestBlockPackingSliced);
+ UNIT_TEST_SUITE_END();
+};
+
+class TMiniKQLComputationNodeTransportFastPackTest: public TMiniKQLComputationNodePackTest<true, true> {
+ UNIT_TEST_SUITE(TMiniKQLComputationNodeTransportFastPackTest);
+ UNIT_TEST(TestNumericTypes);
+ UNIT_TEST(TestOptionalNumericTypes);
+ UNIT_TEST(TestStringTypes);
+ UNIT_TEST(TestUuidType);
+ UNIT_TEST(TestOptionalStringTypes);
+ UNIT_TEST(TestListType);
+ UNIT_TEST(TestListOfOptionalsType);
+ UNIT_TEST(TestTupleType);
+ UNIT_TEST(TestStructType);
+ UNIT_TEST(TestOptionalType);
+ UNIT_TEST(TestDictType);
+ UNIT_TEST(TestVariantTypeOverStruct);
+ UNIT_TEST(TestVariantTypeOverTuple);
+ UNIT_TEST(TestIntegerPackPerformance);
+ UNIT_TEST(TestShortStringPackPerformance);
+ UNIT_TEST(TestPairPackPerformance);
+ UNIT_TEST(TestTuplePackPerformance);
+ UNIT_TEST(TestRopeSplit);
+ UNIT_TEST(TestIncrementalPacking);
+ UNIT_TEST(TestBlockPacking);
+ UNIT_TEST(TestBlockPackingSliced);
+ UNIT_TEST_SUITE_END();
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeGenericPackTest);
+UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeGenericFastPackTest);
+UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeTransportPackTest);
+UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodeTransportFastPackTest);
+}
+}
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp
new file mode 100644
index 00000000000..f16ee88a84e
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp
@@ -0,0 +1,837 @@
+#include "library/cpp/threading/local_executor/local_executor.h"
+#include "ydb/library/yql/minikql/comp_nodes/ut/mkql_computation_node_ut.h"
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_pattern_cache.h>
+#include <ydb/library/yql/minikql/mkql_type_builder.h>
+#include <ydb/library/yql/minikql/mkql_node_serialization.h>
+#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/library/yql/minikql/mkql_node.h>
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
+#include <ydb/library/yql/dq/proto/dq_tasks.pb.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/datetime/cputimer.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+using namespace NYql::NUdf;
+
+TComputationNodeFactory GetListTestFactory() {
+ return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+ if (callable.GetType()->GetName() == "TestList") {
+ return new TExternalComputationNode(ctx.Mutables);
+ }
+ return GetBuiltinFactory()(callable, ctx);
+ };
+}
+
+TRuntimeNode CreateFlow(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ if (list) {
+ return pb.ToFlow(TRuntimeNode(list, false));
+ } else {
+ std::vector<const TRuntimeNode> arr;
+ arr.reserve(vecSize);
+ for (ui64 i = 0; i < vecSize; ++i) {
+ arr.push_back(pb.NewDataLiteral<ui64>((i + 124515) % 6740234));
+ }
+ TArrayRef<const TRuntimeNode> arrRef(std::move(arr));
+ return pb.ToFlow(pb.AsList(arrRef));
+ }
+}
+
+template<bool Wide>
+TRuntimeNode CreateFilter(TProgramBuilder& pb, size_t vecSize, TCallable* list);
+
+template<>
+TRuntimeNode CreateFilter<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ auto handler = [&](TRuntimeNode node) -> TRuntimeNode {
+ return pb.AggrEquals(
+ pb.Mod(node, pb.NewOptional(pb.NewDataLiteral<ui64>(128))),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(0)));
+ };
+ return pb.Filter(flow, handler);
+}
+
+template<>
+TRuntimeNode CreateFilter<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ auto handler = [&](TRuntimeNode::TList node) -> TRuntimeNode {
+ return pb.AggrEquals(
+ pb.Mod(node.front(), pb.NewOptional(pb.NewDataLiteral<ui64>(128))),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(0)));
+ };
+ return pb.NarrowMap(
+ pb.WideFilter(
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ ),
+ handler
+ ),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); }
+ );
+}
+
+template<bool Wide>
+TRuntimeNode CreateMap(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr);
+
+template<>
+TRuntimeNode CreateMap<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ auto handler = [&](TRuntimeNode node) -> TRuntimeNode {
+ return pb.AggrEquals(
+ pb.Mod(node, pb.NewOptional(pb.NewDataLiteral<ui64>(128))),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(0)));
+ };
+ return pb.Map(flow, handler);
+}
+
+template<>
+TRuntimeNode CreateMap<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ auto handler = [&](TRuntimeNode::TList node) -> TRuntimeNode::TList {
+ return {pb.AggrEquals(
+ pb.Mod(node.front(), pb.NewOptional(pb.NewDataLiteral<ui64>(128))),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(0)))};
+ };
+ return pb.NarrowMap(
+ pb.WideMap(
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ ),
+ handler
+ ),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); }
+ );
+}
+
+template<bool Wide>
+TRuntimeNode CreateCondense(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr);
+
+template<>
+TRuntimeNode CreateCondense<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ auto switcherHandler = [&](TRuntimeNode, TRuntimeNode) -> TRuntimeNode {
+ return pb.NewDataLiteral<bool>(false);
+ };
+ auto updateHandler = [&](TRuntimeNode item, TRuntimeNode state) -> TRuntimeNode {
+ return pb.Add(item, state);
+ };
+ TRuntimeNode state = pb.NewDataLiteral<ui64>(0);
+ return pb.Condense(flow, state, switcherHandler, updateHandler);
+}
+
+template<>
+TRuntimeNode CreateCondense<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ TRuntimeNode state = pb.NewDataLiteral<ui64>(0);
+ return pb.NarrowMap(
+ pb.WideCondense1(
+ /* stream */
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ ),
+ /* init */
+ [&](TRuntimeNode::TList item) -> TRuntimeNode::TList { return {item}; },
+ /* switcher */
+ [&](TRuntimeNode::TList, TRuntimeNode::TList) -> TRuntimeNode { return pb.NewDataLiteral<bool>(false); },
+ /* handler */
+ [&](TRuntimeNode::TList item, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.Add(item.front(), state.front())}; }
+ ),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); }
+ );
+}
+
+template<bool Wide>
+TRuntimeNode CreateChopper(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr);
+
+template<>
+TRuntimeNode CreateChopper<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ return pb.Chopper(flow,
+ /* keyExtractor */
+ [&](TRuntimeNode item) -> TRuntimeNode { return item; },
+ /* groupSwitch */
+ [&](TRuntimeNode key, TRuntimeNode /*item*/) -> TRuntimeNode {
+ return pb.AggrEquals(
+ pb.Mod(key, pb.NewOptional(pb.NewDataLiteral<ui64>(128))),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(0)));
+ },
+ /* groupHandler */
+ [&](TRuntimeNode, TRuntimeNode list) -> TRuntimeNode { return list; }
+ );
+};
+
+template<>
+TRuntimeNode CreateChopper<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ return pb.NarrowMap(
+ pb.WideChopper(
+ /* stream */
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ ),
+ /* keyExtractor */
+ [&](TRuntimeNode::TList item) -> TRuntimeNode::TList { return item; },
+ /* groupSwitch */
+ [&](TRuntimeNode::TList key, TRuntimeNode::TList /*item*/) -> TRuntimeNode {
+ return pb.AggrEquals(
+ pb.Mod(key.front(), pb.NewOptional(pb.NewDataLiteral<ui64>(128))),
+ pb.NewOptional(pb.NewDataLiteral<ui64>(0)));
+ },
+ /* groupHandler */
+ [&](TRuntimeNode::TList, TRuntimeNode input) { return pb.WideMap(input, [](TRuntimeNode::TList items) { return items; }); }
+ ),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); }
+ );
+};
+
+template<bool Wide>
+TRuntimeNode CreateCombine(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr);
+
+template<>
+TRuntimeNode CreateCombine<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ return pb.CombineCore(
+ /* stream */
+ flow,
+ /* keyExtractor */
+ [&] (TRuntimeNode /*item*/) -> TRuntimeNode { return pb.NewDataLiteral<ui64>(0);},
+ /* init */
+ [&] (TRuntimeNode /* key */, TRuntimeNode item) -> TRuntimeNode { return item; },
+ /* update */
+ [&] (TRuntimeNode /* key */, TRuntimeNode item, TRuntimeNode state) -> TRuntimeNode { return pb.Add(item, state); },
+ /* finish */
+ [&] (TRuntimeNode /* key */, TRuntimeNode item) -> TRuntimeNode { return pb.NewOptional(item); },
+ /* memlimit */
+ 64 << 20
+ );
+};
+
+template<>
+TRuntimeNode CreateCombine<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ return pb.NarrowMap(
+ pb.WideCombiner(
+ /* stream */
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ ),
+ /* memlimit */
+ 64 << 20,
+ /* keyExtractor */
+ [&] (TRuntimeNode::TList /*item*/) -> TRuntimeNode::TList { return {pb.NewDataLiteral<ui64>(0)};},
+ /* init */
+ [&] (TRuntimeNode::TList /* key */, TRuntimeNode::TList item) -> TRuntimeNode::TList { return {item}; },
+ /* update */
+ [&] (TRuntimeNode::TList /* key */, TRuntimeNode::TList item, TRuntimeNode::TList state) -> TRuntimeNode::TList {
+ return {pb.Add(item.front(), state.front())};
+ },
+ /* finish */
+ [&] (TRuntimeNode::TList /* key */, TRuntimeNode::TList item) -> TRuntimeNode::TList { return {pb.NewOptional(item.front())}; }
+ ),
+ [&](TRuntimeNode::TList items) -> TRuntimeNode { return items.front(); }
+ );
+};
+
+template<bool Wide>
+TRuntimeNode CreateChain1Map(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr);
+
+template<>
+TRuntimeNode CreateChain1Map<false>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ return pb.Chain1Map(
+ flow,
+ /* init */
+ [&] (TRuntimeNode item) -> TRuntimeNode { return item; },
+ /* update */
+ [&] (TRuntimeNode item, TRuntimeNode state) -> TRuntimeNode { return pb.Add(item, state); }
+ );
+}
+
+template<>
+TRuntimeNode CreateChain1Map<true>(TProgramBuilder& pb, size_t vecSize, TCallable* list) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ return pb.NarrowMap(
+ pb.WideChain1Map(
+ /* stream */
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ ),
+ /* init */
+ [&] (TRuntimeNode::TList item) -> TRuntimeNode::TList { return item; },
+ /* update */
+ [&] (TRuntimeNode::TList item, TRuntimeNode::TList state) -> TRuntimeNode::TList { return {pb.Add(item.front(), state.front())}; }
+ ),
+ [&] (TRuntimeNode::TList item) -> TRuntimeNode { return item.front(); }
+ );
+}
+
+template<bool Wide>
+TRuntimeNode CreateDiscard(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ if (Wide) {
+ return pb.Discard(
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ )
+ );
+ } else {
+ return pb.Discard(flow);
+ }
+}
+
+template<bool Wide>
+TRuntimeNode CreateSkip(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ auto count = pb.NewDataLiteral<ui64>(500);
+ if (Wide) {
+ return pb.NarrowMap(
+ pb.Skip(
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ ),
+ count
+ ),
+ [&] (TRuntimeNode::TList item) -> TRuntimeNode { return item.front(); }
+ );
+ } else {
+ return pb.Skip(flow, count);
+ }
+}
+
+template<bool Flow>
+TRuntimeNode CreateNarrowFlatMap(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ return pb.NarrowFlatMap(
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ ),
+ [&] (TRuntimeNode::TList item) -> TRuntimeNode {
+ auto x = pb.NewOptional(item.front());
+ return Flow ? pb.ToFlow(x) : x;
+ }
+ );
+}
+
+TRuntimeNode CreateNarrowMultiMap(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ return pb.NarrowMultiMap(
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ ),
+ [&] (TRuntimeNode::TList item) -> TRuntimeNode::TList {
+ return {item.front(), item.front()};
+ }
+ );
+}
+
+template<bool WithPayload>
+TRuntimeNode CreateSqueezeToSortedDict(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ return pb.FlatMap(
+ pb.NarrowSqueezeToSortedDict(
+ pb.ExpandMap(flow,
+ [&](TRuntimeNode item) -> TRuntimeNode::TList { return {item}; }
+ ),
+ /*all*/ false,
+ /*keySelector*/ [&](TRuntimeNode::TList item) { return item.front(); },
+ /*payloadSelector*/ [&](TRuntimeNode::TList ) { return WithPayload ? pb.NewDataLiteral<ui64>(0) : pb.NewVoid(); }
+ ),
+ [&] (TRuntimeNode item) { return pb.DictKeys(item); }
+ );
+}
+
+TRuntimeNode CreateMapJoin(TProgramBuilder& pb, size_t vecSize, TCallable* list = nullptr) {
+ TTimer t(TString(__func__) + ": ");
+ auto flow = CreateFlow(pb, vecSize, list);
+
+ const auto tupleType = pb.NewTupleType({
+ pb.NewDataType(NUdf::TDataType<ui32>::Id),
+ pb.NewDataType(NUdf::TDataType<ui64>::Id)
+ });
+
+ const auto list1 = pb.Map(flow, [&] (TRuntimeNode item) {
+ return pb.NewTuple({pb.Mod(item, pb.NewDataLiteral<ui64>(1000)), pb.NewDataLiteral<ui32>(1)});
+ });
+
+ const auto list2 = pb.NewList(tupleType, {
+ pb.NewTuple({pb.NewDataLiteral<ui32>(1), pb.NewDataLiteral<ui64>(3 * 1000)}),
+ pb.NewTuple({pb.NewDataLiteral<ui32>(2), pb.NewDataLiteral<ui64>(4 * 1000)}),
+ pb.NewTuple({pb.NewDataLiteral<ui32>(3), pb.NewDataLiteral<ui64>(5 * 1000)}),
+ });
+
+ const auto dict = pb.ToSortedDict(list2, false,
+ [&](TRuntimeNode item) {
+ return pb.Nth(item, 0);
+ },
+ [&](TRuntimeNode item) {
+ return pb.NewTuple({pb.Nth(item, 1U)});
+ });
+
+ const auto resultType = pb.NewFlowType(pb.NewMultiType({
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ pb.NewDataType(NUdf::TDataType<char*>::Id),
+ }));
+
+ return pb.Map(
+ pb.NarrowMap(pb.MapJoinCore(
+ pb.ExpandMap(list1, [&] (TRuntimeNode item) -> TRuntimeNode::TList { return {pb.Nth(item, 0), pb.Nth(item, 1)}; }),
+ dict,
+ EJoinKind::Inner,
+ {0U},
+ {1U, 0U},
+ {0U, 1U},
+ resultType
+ ),
+ [&](TRuntimeNode::TList items) { return pb.NewTuple(items); }
+ ),
+ [&](TRuntimeNode item) { return pb.Nth(item, 1); }
+ );
+}
+
+Y_UNIT_TEST_SUITE(ComputationGraphDataRace) {
+ template<class T>
+ void ParallelProgTest(T f, bool useLLVM, ui64 testResult, size_t vecSize = 10'000) {
+ TTimer t("total: ");
+ const ui32 cacheSizeInBytes = 104857600; // 100 MiB
+ const ui32 inFlight = 7;
+ TComputationPatternLRUCache cache(cacheSizeInBytes);
+
+ auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry())->Clone();
+ auto entry = std::make_shared<TPatternCacheEntry>();
+ TScopedAlloc& alloc = entry->Alloc;
+ TTypeEnvironment& typeEnv = entry->Env;
+
+ TProgramBuilder pb(typeEnv, *functionRegistry);
+
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<ui64>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+ TRuntimeNode progReturn;
+ with_lock(alloc) {
+ progReturn = f(pb, vecSize, list);
+ }
+
+ TExploringNodeVisitor explorer;
+ explorer.Walk(progReturn.GetNode(), typeEnv);
+
+ TComputationPatternOpts opts(alloc.Ref(), typeEnv, GetListTestFactory(), functionRegistry.Get(),
+ NUdf::EValidateMode::Lazy, NUdf::EValidatePolicy::Exception, useLLVM ? "" : "OFF", EGraphPerProcess::Multi);
+
+ {
+ auto guard = entry->Env.BindAllocator();
+ entry->Pattern = MakeComputationPattern(explorer, progReturn, {list}, opts);
+ }
+ cache.EmplacePattern("a", entry);
+ auto genData = [&]() {
+ std::vector<ui64> data;
+ data.reserve(vecSize);
+ for (ui64 i = 0; i < vecSize; ++i) {
+ data.push_back((i + 124515) % 6740234);
+ }
+ return data;
+ };
+
+ const auto data = genData();
+
+ std::vector<std::vector<ui64>> results(inFlight);
+
+ NPar::LocalExecutor().RunAdditionalThreads(inFlight);
+ NPar::LocalExecutor().ExecRange([&](int id) {
+ for (ui32 i = 0; i < 100; ++i) {
+ auto key = "a";
+
+ auto randomProvider = CreateDeterministicRandomProvider(1);
+ auto timeProvider = CreateDeterministicTimeProvider(10000000);
+ TScopedAlloc graphAlloc(__LOCATION__);
+
+ auto entry = cache.Find(key);
+
+ TComputationPatternOpts opts(entry->Alloc.Ref(), entry->Env, GetListTestFactory(),
+ functionRegistry.Get(), NUdf::EValidateMode::Lazy, NUdf::EValidatePolicy::Exception,
+ useLLVM ? "" : "OFF", EGraphPerProcess::Multi);
+
+ auto graph = entry->Pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider, &graphAlloc.Ref()));
+ TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(data.size(), items));
+
+ std::transform(data.cbegin(), data.cend(), items,
+ [](const auto s) {
+ return ToValue<ui64>(s);
+ });
+
+ ui64 acc = 0;
+ TUnboxedValue v = graph->GetValue();
+ while (v.HasValue()) {
+ acc += v.Get<ui64>();
+ v = graph->GetValue();
+ }
+ results[id].push_back(acc);
+ }
+ }, 0, inFlight, NPar::TLocalExecutor::WAIT_COMPLETE | NPar::TLocalExecutor::MED_PRIORITY);
+
+ for (auto threadResults : results) {
+ for (auto res : threadResults) {
+ UNIT_ASSERT_VALUES_EQUAL(res, testResult);
+ }
+ }
+ }
+
+ Y_UNIT_TEST_QUAD(Filter, Wide, UseLLVM) {
+ ParallelProgTest(CreateFilter<Wide>, UseLLVM, 10098816);
+ }
+
+ Y_UNIT_TEST_QUAD(Map, Wide, UseLLVM) {
+ ParallelProgTest(CreateMap<Wide>, UseLLVM, 78);
+ }
+
+ Y_UNIT_TEST_QUAD(Condense, Wide, UseLLVM) {
+ ParallelProgTest(CreateCondense<Wide>, UseLLVM, 1295145000);
+ }
+
+ Y_UNIT_TEST_QUAD(Chopper, Wide, UseLLVM) {
+ ParallelProgTest(CreateChopper<Wide>, UseLLVM, 1295145000);
+ }
+
+ Y_UNIT_TEST_QUAD(Combine, Wide, UseLLVM) {
+ ParallelProgTest(CreateCombine<Wide>, UseLLVM, 1295145000);
+ }
+
+ Y_UNIT_TEST_QUAD(Chain1Map, Wide, UseLLVM) {
+ ParallelProgTest(CreateChain1Map<Wide>, UseLLVM, 6393039240000);
+ }
+
+ Y_UNIT_TEST_QUAD(Discard, Wide, UseLLVM) {
+ ParallelProgTest(CreateDiscard<Wide>, UseLLVM, 0);
+ }
+
+ Y_UNIT_TEST_QUAD(Skip, Wide, UseLLVM) {
+ ParallelProgTest(CreateSkip<Wide>, UseLLVM, 1232762750);
+ }
+
+ Y_UNIT_TEST_QUAD(NarrowFlatMap, Flow, UseLLVM) {
+ ParallelProgTest(CreateNarrowFlatMap<Flow>, UseLLVM, 1295145000);
+ }
+
+ Y_UNIT_TEST_TWIN(NarrowMultiMap, UseLLVM) {
+ ParallelProgTest(CreateNarrowMultiMap, UseLLVM, 1295145000ull * 2);
+ }
+
+ Y_UNIT_TEST_QUAD(SqueezeToSortedDict, WithPayload, UseLLVM) {
+ ParallelProgTest(CreateSqueezeToSortedDict<WithPayload>, UseLLVM, 125014500, 1000);
+ }
+
+ Y_UNIT_TEST_TWIN(MapJoin, UseLLVM) {
+ ParallelProgTest(CreateMapJoin, UseLLVM, 120000, 10'000);
+ }
+}
+
+
+Y_UNIT_TEST_SUITE(ComputationPatternCache) {
+ Y_UNIT_TEST(Smoke) {
+ const ui32 cacheSize = 10'000'000;
+ const ui32 cacheItems = 10;
+ TComputationPatternLRUCache cache(cacheSize);
+
+ auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry())->Clone();
+
+ for (ui32 i = 0; i < cacheItems; ++i) {
+ auto entry = std::make_shared<TPatternCacheEntry>();
+ TScopedAlloc& alloc = entry->Alloc;
+ TTypeEnvironment& typeEnv = entry->Env;
+
+ TProgramBuilder pb(typeEnv, *functionRegistry);
+
+ TRuntimeNode progReturn;
+ with_lock(alloc) {
+ progReturn = pb.NewDataLiteral<NYql::NUdf::EDataSlot::String>("qwerty");
+ }
+
+ TExploringNodeVisitor explorer;
+ explorer.Walk(progReturn.GetNode(), typeEnv);
+
+ TComputationPatternOpts opts(alloc.Ref(), typeEnv, GetBuiltinFactory(),
+ functionRegistry.Get(), NUdf::EValidateMode::Lazy, NUdf::EValidatePolicy::Exception,
+ "OFF", EGraphPerProcess::Multi);
+
+ {
+ auto guard = entry->Env.BindAllocator();
+ entry->Pattern = MakeComputationPattern(explorer, progReturn, {}, opts);
+ }
+ cache.EmplacePattern(TString((char)('a' + i)), entry);
+ }
+
+ for (ui32 i = 0; i < cacheItems; ++i) {
+ auto key = TString((char)('a' + i));
+
+ auto randomProvider = CreateDeterministicRandomProvider(1);
+ auto timeProvider = CreateDeterministicTimeProvider(10000000);
+ TScopedAlloc graphAlloc(__LOCATION__);
+ auto entry = cache.Find(key);
+ UNIT_ASSERT(entry);
+ TComputationPatternOpts opts(entry->Alloc.Ref(), entry->Env, GetBuiltinFactory(),
+ functionRegistry.Get(), NUdf::EValidateMode::Lazy, NUdf::EValidatePolicy::Exception,
+ "OFF", EGraphPerProcess::Multi);
+
+ auto graph = entry->Pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider, &graphAlloc.Ref()));
+ auto value = graph->GetValue();
+ UNIT_ASSERT_EQUAL(value.AsStringRef(), NYql::NUdf::TStringRef("qwerty"));
+ }
+ }
+
+ Y_UNIT_TEST(AddPerf) {
+ TTimer t("all: ");
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment typeEnv(alloc);
+
+ auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry())->Clone();
+
+ TProgramBuilder pb(typeEnv, *functionRegistry);
+ auto prog1 = pb.NewDataLiteral<ui64>(123591592ULL);
+ auto prog2 = pb.NewDataLiteral<ui64>(323591592ULL);
+ auto progReturn = pb.Add(prog1, prog2);
+
+ TExploringNodeVisitor explorer;
+ explorer.Walk(progReturn.GetNode(), typeEnv);
+
+ NUdf::EValidateMode validateMode = NUdf::EValidateMode::Lazy;
+ TComputationPatternOpts opts(alloc.Ref(), typeEnv, GetBuiltinFactory(),
+ functionRegistry.Get(), validateMode, NUdf::EValidatePolicy::Exception,
+ "OFF", EGraphPerProcess::Multi);
+
+ auto t_make_pattern = std::make_unique<TTimer>("make_pattern: ");
+ auto pattern = MakeComputationPattern(explorer, progReturn, {}, opts);
+ t_make_pattern.reset();
+ auto randomProvider = CreateDeterministicRandomProvider(1);
+ auto timeProvider = CreateDeterministicTimeProvider(10000000);
+
+ auto t_clone = std::make_unique<TTimer>("clone: ");
+ auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider));
+ t_clone.reset();
+
+ const ui64 repeats = 100'000;
+
+ {
+ TTimer t("graph: ");
+ ui64 acc = 0;
+ for (ui64 i = 0; i < repeats; ++i) {
+ acc += graph->GetValue().Get<ui64>();
+ }
+ Y_DO_NOT_OPTIMIZE_AWAY(acc);
+ }
+ {
+ std::function<ui64(ui64, ui64)> add = [](ui64 a, ui64 b) {
+ return a + b;
+ };
+
+ TTimer t("lambda: ");
+ ui64 acc = 0;
+ for (ui64 i = 0; i < repeats; ++i) {
+ acc += add(123591592ULL, 323591592ULL);
+ }
+ Y_DO_NOT_OPTIMIZE_AWAY(acc);
+ }
+ {
+ std::function<TUnboxedValue(TUnboxedValue&, TUnboxedValue&)> add =
+ [](TUnboxedValue& a, TUnboxedValue& b) {
+ return TUnboxedValuePod(a.Get<ui64>() + b.Get<ui64>());
+ };
+ Y_DO_NOT_OPTIMIZE_AWAY(add);
+
+ TTimer t("lambda unboxed value: ");
+ TUnboxedValue acc(TUnboxedValuePod(0));
+ TUnboxedValue v1(TUnboxedValuePod(ui64{123591592UL}));
+ TUnboxedValue v2(TUnboxedValuePod(ui64{323591592UL}));
+ for (ui64 i = 0; i < repeats; ++i) {
+ auto r = add(v1, v2);
+ acc = add(r, acc);
+ }
+ Y_DO_NOT_OPTIMIZE_AWAY(acc.Get<ui64>());
+ }
+ }
+
+ Y_UNIT_TEST_TWIN(FilterPerf, Wide) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment typeEnv(alloc);
+
+ auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry())->Clone();
+
+ TProgramBuilder pb(typeEnv, *functionRegistry);
+ const ui64 vecSize = 100'000;
+ Cerr << "vecSize: " << vecSize << Endl;
+ const auto listType = pb.NewListType(pb.NewDataType(NUdf::TDataType<ui64>::Id));
+ const auto list = TCallableBuilder(pb.GetTypeEnvironment(), "TestList", listType).Build();
+ auto progReturn = CreateFilter<Wide>(pb, vecSize, list);
+
+ TExploringNodeVisitor explorer;
+ explorer.Walk(progReturn.GetNode(), typeEnv);
+
+ NUdf::EValidateMode validateMode = NUdf::EValidateMode::Max;
+ TComputationPatternOpts opts(alloc.Ref(), typeEnv, GetListTestFactory(),
+ functionRegistry.Get(), validateMode, NUdf::EValidatePolicy::Exception,
+ "OFF", EGraphPerProcess::Multi);
+
+ auto t_make_pattern = std::make_unique<TTimer>("make_pattern: ");
+ auto pattern = MakeComputationPattern(explorer, progReturn, {list}, opts);
+ t_make_pattern.reset();
+
+ auto randomProvider = CreateDeterministicRandomProvider(1);
+ auto timeProvider = CreateDeterministicTimeProvider(10000000);
+
+ auto t_clone = std::make_unique<TTimer>("clone: ");
+ auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider));
+
+ t_clone.reset();
+
+ auto genData = [&]() {
+ std::vector<ui64> data;
+ data.reserve(vecSize);
+ for (ui64 i = 0; i < vecSize; ++i) {
+ data.push_back((i + 124515) % 6740234);
+ }
+ return data;
+ };
+
+ auto testResult = [&] (ui64 acc, ui64 count) {
+ if (vecSize == 100'000'000) {
+ UNIT_ASSERT_VALUES_EQUAL(acc, 2614128386688);
+ UNIT_ASSERT_VALUES_EQUAL(count, 781263);
+ } else if (vecSize == 10'000'000) {
+ UNIT_ASSERT_VALUES_EQUAL(acc, 222145217664);
+ } else if (vecSize == 100'000) {
+ UNIT_ASSERT_VALUES_EQUAL(acc, 136480896);
+ UNIT_ASSERT_VALUES_EQUAL(count, 782);
+ } else {
+ UNIT_FAIL("result is not checked");
+ }
+ };
+
+ ui64 kIter = 2;
+ {
+ TDuration total;
+ for (ui64 i = 0; i < kIter; ++i) {
+ ui64 acc = 0;
+ ui64 count = 0;
+
+ auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider));
+ auto data = genData();
+ TUnboxedValue* items = nullptr;
+ graph->GetEntryPoint(0, true)->SetValue(graph->GetContext(), graph->GetHolderFactory().CreateDirectArrayHolder(data.size(), items));
+
+ std::transform(data.cbegin(), data.cend(), items,
+ [](const auto s) {
+ return ToValue<ui64>(s);
+ });
+
+ TSimpleTimer t;
+ TUnboxedValue v = graph->GetValue();
+ while (v.HasValue()) {
+ acc += v.Get<ui64>();
+ ++count;
+ v = graph->GetValue();
+ }
+ testResult(acc, count);
+
+ total += t.Get();
+ }
+ Cerr << "graph: " << Sprintf("%.3f", total.SecondsFloat()) << "s" << Endl;
+ }
+
+ {
+ auto data = genData();
+ std::function<bool(ui64)> predicate = [](ui64 a) {
+ return a % 128 == 0;
+ };
+ Y_DO_NOT_OPTIMIZE_AWAY(predicate);
+
+ TDuration total;
+
+ for (ui64 i = 0; i < kIter; ++i) {
+ TSimpleTimer t;
+ ui64 acc = 0;
+ ui64 count = 0;
+ for (ui64 j = 0; j < data.size(); ++j) {
+ if (predicate(data[j])) {
+ acc += data[j];
+ ++count;
+ }
+ }
+
+ total += t.Get();
+
+ testResult(acc, count);
+ }
+ Cerr << "std::function: " << Sprintf("%.3f", total.SecondsFloat()) << "s" << Endl;
+ }
+
+ {
+ auto data = genData();
+ auto predicate = [](ui64 a) {
+ return a % 128 == 0;
+ };
+ Y_DO_NOT_OPTIMIZE_AWAY(predicate);
+
+ TDuration total;
+ for (ui64 i = 0; i < kIter; ++i) {
+ TSimpleTimer t;
+ ui64 acc = 0;
+ ui64 count = 0;
+ for (ui64 j = 0; j < data.size(); ++j) {
+ if (predicate(data[j])) {
+ acc += data[j];
+ ++count;
+ }
+ }
+
+ total += t.Get();
+
+ testResult(acc, count);
+ }
+ Cerr << "lambda: " << Sprintf("%.3f", total.SecondsFloat()) << "s" << Endl;
+ }
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp b/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp
new file mode 100644
index 00000000000..2dd6ec95e6c
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp
@@ -0,0 +1,1153 @@
+#include <ydb/library/yql/minikql/mkql_program_builder.h>
+#include <ydb/library/yql/minikql/mkql_node_printer.h>
+#include "mkql_computation_list_adapter.h"
+#include "mkql_computation_node_impl.h"
+#include "mkql_computation_node.h"
+#include "mkql_value_builder.h"
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include "mkql_validate.h"
+
+#include <ydb/library/yql/minikql/mkql_type_builder.h>
+#include <ydb/library/yql/minikql/mkql_utils.h>
+#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/generic/algorithm.h>
+
+#include <ydb/library/yql/public/udf/udf_helpers.h>
+
+namespace NYql {
+
+namespace {
+using namespace NKikimr::NMiniKQL;
+static const ui32 RAW_INDEX_NO_HOLE = -1;
+static const ui32 RAW_BROKEN_INDEX_LIST_TO_DICT = 1;
+
+template<class T>
+NUdf::TUnboxedValue ToUnboxedValue(const T& val) {
+ return NUdf::TUnboxedValuePod(val);
+}
+
+NUdf::TUnboxedValue ToUnboxedValue(const TString& val) {
+ return MakeString(val);
+}
+
+NUdf::TUnboxedValue ToUnboxedValue(const NUdf::IBoxedValuePtr& val) {
+ return NUdf::TUnboxedValuePod(NUdf::IBoxedValuePtr(val));
+}
+
+} // namespace NMiniKQL
+
+/// support for build Struct type @{
+namespace NUdf {
+
+ template<class TContainer>
+ struct TListRefIterator: public TBoxedValue {
+ TListRefIterator(const TContainer& listRef, ui32 holePos)
+ : ListRef(listRef)
+ , Index(-1)
+ , HolePos(holePos)
+ {}
+ private:
+ const TContainer& ListRef;
+ ui32 Index;
+ ui32 HolePos;
+
+ bool Next(NUdf::TUnboxedValue& value) final {
+ if (++Index >= ListRef.size())
+ return false;
+ value = Index == HolePos ? NUdf::TUnboxedValue(NUdf::TUnboxedValuePod(42)) : ToUnboxedValue(ListRef[Index]);
+ return true;
+ }
+ };
+
+ template<class TContainer, ui32 TIndexDictBrokenHole = RAW_INDEX_NO_HOLE, bool TNoDictIndex = false>
+ struct TListRef: public NUdf::TBoxedValue {
+ TListRef(const TContainer& listRef, ui32 holePos = RAW_INDEX_NO_HOLE)
+ : ListRef(listRef)
+ , HolePos(holePos)
+ {}
+
+ private:
+ const TContainer& ListRef;
+ const NUdf::IValueBuilder* ValueBuilder;
+ ui32 HolePos;
+
+ bool HasFastListLength() const override {
+ return true;
+ }
+
+ ui64 GetListLength() const override {
+ return ListRef.size();
+ }
+
+ ui64 GetEstimatedListLength() const override {
+ return ListRef.size();
+ }
+
+ NUdf::TUnboxedValue GetListIterator() const override {
+ return NUdf::TUnboxedValuePod(new TListRefIterator<TContainer>(ListRef, HolePos));
+ }
+
+ NUdf::IBoxedValuePtr ToIndexDictImpl(const IValueBuilder& builder) const override {
+ return TNoDictIndex ? nullptr : builder.ToIndexDict(NUdf::TUnboxedValuePod(
+ new TListRef<TContainer, TIndexDictBrokenHole, true>(ListRef, TIndexDictBrokenHole))).AsBoxed();
+ }
+ };
+
+ struct PersonStruct {
+ static const size_t MEMBERS_COUNT = 3;
+ static ui32 MetaIndexes[MEMBERS_COUNT];
+ static ui32 MetaBackIndexes[MEMBERS_COUNT];
+
+ TString FirstName;
+ TString LastName;
+ ui32 Age;
+
+ NUdf::TUnboxedValue GetByIndex(ui32 index) const {
+ switch (index) {
+ case 0: return ToUnboxedValue(FirstName);
+ case 1: return ToUnboxedValue(LastName);
+ case 2: return NUdf::TUnboxedValuePod(Age);
+ default: Y_FAIL("Unexpected");
+ }
+ }
+ };
+
+ ui32 PersonStruct::MetaIndexes[MEMBERS_COUNT];
+ ui32 PersonStruct::MetaBackIndexes[MEMBERS_COUNT];
+
+
+ struct PersonStructWithOptList {
+ static const size_t MEMBERS_COUNT = 4;
+ static ui32 MetaIndexes[MEMBERS_COUNT];
+ static ui32 MetaBackIndexes[MEMBERS_COUNT];
+
+ TString FirstName;
+ TString LastName;
+ ui32 Age;
+ typedef std::vector<ui32> TTagList;
+ TTagList Tags;
+
+ NUdf::TUnboxedValue GetByIndex(ui32 index) const {
+ switch (index) {
+ case 0: return ToUnboxedValue(FirstName);
+ case 1: return ToUnboxedValue(LastName);
+ case 2: return NUdf::TUnboxedValuePod(Age);
+ case 3: return Tags.empty() ?
+ NUdf::TUnboxedValuePod() :
+ NUdf::TUnboxedValuePod(new TListRef<TTagList>(Tags));
+ default: Y_FAIL("Unexpected");
+ }
+ }
+ };
+
+ ui32 PersonStructWithOptList::MetaIndexes[MEMBERS_COUNT];
+ ui32 PersonStructWithOptList::MetaBackIndexes[MEMBERS_COUNT];
+
+ struct TCallableOneUi32Arg {
+ };
+
+namespace NImpl {
+
+ template<>
+ struct TTypeBuilderHelper<NUdf::PersonStruct> {
+ static TType* Build(const IFunctionTypeInfoBuilder& builder) {
+ auto structBuilder = builder.Struct(3);
+ structBuilder->AddField<char*>("FirstName", &PersonStruct::MetaIndexes[0])
+ .AddField<char*>("LastName", &PersonStruct::MetaIndexes[1])
+ .AddField<ui32>("Age", &PersonStruct::MetaIndexes[2]);
+ auto structType = structBuilder->Build();
+ for (const auto& index: PersonStruct::MetaIndexes) {
+ Y_VERIFY(index < NUdf::PersonStruct::MEMBERS_COUNT);
+ NUdf::PersonStruct::MetaBackIndexes[index] = &index - PersonStruct::MetaIndexes;
+ Y_VERIFY(NUdf::PersonStruct::MetaBackIndexes[index] < NUdf::PersonStruct::MEMBERS_COUNT);
+ }
+ return structType;
+ }
+ };
+
+ template<>
+ struct TTypeBuilderHelper<NUdf::PersonStructWithOptList> {
+ static TType* Build(const IFunctionTypeInfoBuilder& builder) {
+ auto listTags = builder.List()->Item<ui32>().Build();
+ auto optionalListTags = builder.Optional()->Item(listTags).Build();
+ auto structBuilder = builder.Struct(3);
+ structBuilder->AddField<char*>("FirstName", &PersonStructWithOptList::MetaIndexes[0])
+ .AddField<char*>("LastName", &PersonStructWithOptList::MetaIndexes[1])
+ .AddField<ui32>("Age", &PersonStructWithOptList::MetaIndexes[2])
+ .AddField("Tags", optionalListTags, &PersonStructWithOptList::MetaIndexes[3]);
+ auto structType = structBuilder->Build();
+ for (const auto& index: PersonStructWithOptList::MetaIndexes) {
+ Y_VERIFY(index < NUdf::PersonStructWithOptList::MEMBERS_COUNT);
+ NUdf::PersonStructWithOptList::MetaBackIndexes[index] = &index - PersonStructWithOptList::MetaIndexes;
+ Y_VERIFY(NUdf::PersonStructWithOptList::MetaBackIndexes[index] < NUdf::PersonStructWithOptList::MEMBERS_COUNT);
+ }
+ return structType;
+ }
+ };
+
+ template<>
+ struct TTypeBuilderHelper<NUdf::TCallableOneUi32Arg> {
+ static TType* Build(const IFunctionTypeInfoBuilder& builder) {
+ auto callableBuilder = builder.Callable(1);
+ callableBuilder->Returns<ui32>();
+ callableBuilder->Arg<ui32>();
+ return callableBuilder->Build();
+ }
+ };
+} // namespace NImpl
+} // namespace NUdf
+/// @}
+
+
+ struct TBrokenSeqListIterator: public NUdf::TBoxedValue {
+ TBrokenSeqListIterator(ui32 size, ui32 holePos)
+ : Size(size)
+ , HolePos(holePos)
+ , Index(-1)
+ {}
+ private:
+ ui32 Size;
+ ui32 HolePos;
+ ui32 Index;
+
+ bool Skip() final {
+ return ++Index < Size;
+ }
+
+ bool Next(NUdf::TUnboxedValue& value) final {
+ if (!Skip())
+ return false;
+ value = Index == HolePos ? NUdf::TUnboxedValuePod() : NUdf::TUnboxedValuePod(Index);
+ return true;
+ }
+ };
+
+ struct TBrokenSeqListBoxedValue: public NUdf::TBoxedValue {
+ TBrokenSeqListBoxedValue(ui32 size, ui32 holePos)
+ : ListSize(size)
+ , HolePos(holePos)
+ {}
+
+ private:
+ ui32 ListSize;
+ ui32 HolePos;
+
+ bool HasFastListLength() const override {
+ return true;
+ }
+
+ ui64 GetListLength() const override {
+ return ListSize;
+ }
+
+ ui64 GetEstimatedListLength() const override {
+ return ListSize;
+ }
+
+ NUdf::TUnboxedValue GetListIterator() const override {
+ return NUdf::TUnboxedValuePod(new TBrokenSeqListIterator(ListSize, HolePos));
+ }
+ };
+
+ template<class TStructType>
+ struct TBrokenStructBoxedValue: public NUdf::TBoxedValue {
+ TBrokenStructBoxedValue(const TStructType& data, ui32 holePos = RAW_INDEX_NO_HOLE)
+ : Struct(data)
+ , HolePos(holePos)
+ {}
+
+ private:
+ const TStructType& Struct;
+ ui32 HolePos;
+
+ NUdf::TUnboxedValue GetElement(ui32 index) const override {
+ if (index == HolePos) {
+ return NUdf::TUnboxedValuePod();
+ }
+ return Struct.GetByIndex(TStructType::MetaBackIndexes[index]);
+ }
+ };
+
+
+namespace {
+ template<>
+ NUdf::TUnboxedValue ToUnboxedValue<NUdf::PersonStruct>(const NUdf::PersonStruct& val) {
+ return NUdf::TUnboxedValuePod(new TBrokenStructBoxedValue<NUdf::PersonStruct>(val));
+ }
+
+ template<>
+ NUdf::TUnboxedValue ToUnboxedValue<NUdf::PersonStructWithOptList>(const NUdf::PersonStructWithOptList& val) {
+ return NUdf::TUnboxedValuePod(new TBrokenStructBoxedValue<NUdf::PersonStructWithOptList>(val));
+ }
+
+ template<class TTupleType>
+ struct TBrokenTupleBoxedValue: public NUdf::TBoxedValue {
+ TBrokenTupleBoxedValue(const TTupleType& tuple, ui32 holePos)
+ : Tuple(tuple)
+ , HolePos(holePos)
+ {}
+
+ private:
+ const TTupleType& Tuple;
+ ui32 HolePos;
+
+ NUdf::TUnboxedValue GetElement(ui32 index) const override {
+ if (index == HolePos) {
+ return NUdf::TUnboxedValuePod();
+ }
+ switch (index) {
+ case 0: return ToUnboxedValue(std::get<0>(Tuple));
+ case 1: return ToUnboxedValue(std::get<1>(Tuple));
+ case 2: return ToUnboxedValue(std::get<2>(Tuple));
+ case 3: return ToUnboxedValue(std::get<3>(Tuple));
+ default: Y_FAIL("Unexpected");
+ }
+ }
+ };
+
+ typedef std::pair<ui32, ui32> PosPair;
+
+ template<class TKey, class TValue>
+ struct TBrokenDictIterator: public NUdf::TBoxedValue {
+ TBrokenDictIterator(const std::vector<std::pair<TKey, TValue>>& dictData, PosPair holePos)
+ : DictData(dictData)
+ , HolePos(holePos)
+ , Index(-1)
+ {}
+
+ private:
+ const std::vector<std::pair<TKey, TValue>>& DictData;
+ PosPair HolePos;
+ ui32 Index;
+
+ bool Skip() final {
+ return ++Index < DictData.size();
+ }
+
+ bool Next(NUdf::TUnboxedValue& key) final {
+ if (!Skip())
+ return false;
+ key = Index == HolePos.first ? NUdf::TUnboxedValuePod() : NUdf::TUnboxedValuePod(DictData[Index].first);
+ return true;
+ }
+
+ bool NextPair(NUdf::TUnboxedValue& key, NUdf::TUnboxedValue& payload) final {
+ if (!Next(key))
+ return false;
+ payload = Index == HolePos.second ? NUdf::TUnboxedValue() : ToUnboxedValue(DictData[Index].second);
+ return true;
+ }
+ };
+
+ template<class TKey, class TValue>
+ struct TBrokenDictBoxedValue: public NUdf::TBoxedValue {
+ TBrokenDictBoxedValue(const std::vector<std::pair<TKey, TValue>>& dictData,
+ PosPair holePos, NUdf::TUnboxedValue&& hole = NUdf::TUnboxedValuePod())
+ : DictData(dictData)
+ , HolePos(holePos)
+ , Hole(std::move(hole))
+ {}
+
+ private:
+ const std::vector<std::pair<TKey, TValue>> DictData;
+ PosPair HolePos;
+ NUdf::TUnboxedValue Hole;
+
+ NUdf::TUnboxedValue GetKeysIterator() const override {
+ return NUdf::TUnboxedValuePod(new TBrokenDictIterator<TKey, TValue>(DictData, HolePos));
+ }
+
+ NUdf::TUnboxedValue GetDictIterator() const override {
+ return NUdf::TUnboxedValuePod(new TBrokenDictIterator<TKey, TValue>(DictData, HolePos));
+ }
+ };
+
+ struct TThrowerValue: public NUdf::TBoxedValue {
+ static long Count;
+ TThrowerValue(NUdf::IBoxedValuePtr&& owner = NUdf::IBoxedValuePtr())
+ : Owner(std::move(owner))
+ { ++Count; }
+ ~TThrowerValue() { --Count; }
+ private:
+ const NUdf::IBoxedValuePtr Owner;
+
+ bool Skip() override {
+ ythrow yexception() << "Throw";
+ }
+
+ NUdf::TUnboxedValue GetListIterator() const override {
+ return NUdf::TUnboxedValuePod(new TThrowerValue(const_cast<TThrowerValue*>(this)));
+ }
+ };
+
+ SIMPLE_UDF(TException, NUdf::TListType<ui32>()) {
+ Y_UNUSED(valueBuilder);
+ Y_UNUSED(args);
+ return NUdf::TUnboxedValuePod(new TThrowerValue);
+ }
+
+ long TThrowerValue::Count = 0L;
+
+ SIMPLE_UDF(TVoid, void()) {
+ Y_UNUSED(valueBuilder);
+ Y_UNUSED(args);
+ return NUdf::TUnboxedValuePod::Void();
+ }
+
+ SIMPLE_UDF_RUN(TNonEmpty, ui32(), NUdf::TOptional<void>) {
+ Y_UNUSED(valueBuilder);
+ Y_UNUSED(args);
+ return NUdf::TUnboxedValuePod(42);
+ }
+
+ SIMPLE_UDF(TOptionalNonEmpty, NUdf::TOptional<ui32>()) {
+ Y_UNUSED(valueBuilder);
+ Y_UNUSED(args);
+ return NUdf::TUnboxedValuePod(42);
+ }
+
+ SIMPLE_UDF_RUN(TOptionalEmpty, NUdf::TOptional<ui32>(), NUdf::TOptional<void>) {
+ Y_UNUSED(valueBuilder);
+ Y_UNUSED(args);
+ return NUdf::TUnboxedValuePod();
+ }
+
+ SIMPLE_UDF(TSub2Mul2BrokenOnLess2, ui32(ui32)) {
+ Y_UNUSED(valueBuilder);
+ const ui32 arg = args[0].Get<ui32>();
+ if (arg >= 2) {
+ return NUdf::TUnboxedValuePod((arg - 2) * 2);
+ }
+ return NUdf::TUnboxedValuePod();
+ }
+
+ SIMPLE_UDF_RUN(TBackSub2Mul2, ui32(NUdf::TCallableOneUi32Arg, ui32), NUdf::TOptional<void>) {
+ const auto func = args[0];
+ const auto& arg = args[1];
+ auto usedArg = NUdf::TUnboxedValuePod();
+ if (arg.Get<ui32>() < 100) {
+ usedArg = arg;
+ }
+ auto funcResult = func.Run(valueBuilder, &usedArg);
+ const auto& backResult = funcResult.Get<ui32>() / 2 + 2;
+ return NUdf::TUnboxedValuePod(backResult);
+ }
+
+ SIMPLE_UDF(TSeqList, NUdf::TListType<ui32>(ui32)) {
+ const ui32 size = args[0].Get<ui32>();
+ std::vector<NUdf::TUnboxedValue> res;
+ res.resize(size);
+ for (ui32 i = 0; i < size; ++i) {
+ res[i] = NUdf::TUnboxedValuePod(i);
+ }
+ return valueBuilder->NewList(res.data(), res.size());
+ }
+
+ SIMPLE_UDF_RUN(TSeqListWithHole, NUdf::TListType<ui32>(ui32, ui32), NUdf::TOptional<void>) {
+ Y_UNUSED(valueBuilder);
+ const ui32 size = args[0].Get<ui32>();
+ const ui32 hole = args[1].Get<ui32>();
+ NUdf::IBoxedValuePtr boxed(new TBrokenSeqListBoxedValue(size, hole));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ static const auto TUPLE = std::make_tuple(ui8(33), TString("world"), ui64(0xFEEDB00B2A115E), TString("funny bunny"));
+
+ typedef NUdf::TTuple<ui8, char*, ui64, char*> NUdfTuple;
+
+ SIMPLE_UDF(TTuple, NUdfTuple(ui32)) {
+ Y_UNUSED(valueBuilder);
+ const ui32 holePos = args[0].Get<ui32>();
+ NUdf::IBoxedValuePtr boxed(new TBrokenTupleBoxedValue<decltype(TUPLE)>(TUPLE, holePos));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ static const std::vector<std::pair<ui32, ui64>> DICT_DIGIT2DIGIT = {
+ {1, 100500},
+ {42, 0xDEADBEAF},
+ {911, 1234567890},
+ {777, 777777777777},
+ };
+
+ typedef NUdf::TDict<ui32, ui64> NUdfDictDigDig;
+
+ SIMPLE_UDF_RUN(TDictDigDig, NUdfDictDigDig(ui32, ui32), NUdf::TOptional<void>) {
+ Y_UNUSED(valueBuilder);
+ const ui32 holeKey = args[0].Get<ui32>();
+ const ui32 holeValue = args[1].Get<ui32>();
+ NUdf::IBoxedValuePtr boxed(new TBrokenDictBoxedValue<ui32, ui64>(
+ DICT_DIGIT2DIGIT, std::make_pair(holeKey, holeValue)));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ SIMPLE_UDF(TDictDigDigHoleAsOpt, NUdfDictDigDig(ui32, ui32)) {
+ Y_UNUSED(valueBuilder);
+ const ui32 holeKey = args[0].Get<ui32>();
+ const ui32 holeValue = args[1].Get<ui32>();
+ NUdf::IBoxedValuePtr boxed(new TBrokenDictBoxedValue<ui32, ui64>(DICT_DIGIT2DIGIT,
+ std::make_pair(holeKey, holeValue),
+ NUdf::TUnboxedValuePod()));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ static const NUdf::PersonStruct STRUCT_PERSON_JONNIE = {"Johnnie Walker", "Blue Label", 25};
+ static const NUdf::PersonStruct STRUCT_PERSON_HITHCOCK = {"Alfred", "Hithcock", 81};
+ static const NUdf::PersonStruct STRUCT_PERSON_LOVECRAFT = {"Howard", "Lovecraft", 25};
+ static const NUdf::PersonStruct STRUCT_PERSON_KING = {"Stephen", "King", 25};
+ static const NUdf::PersonStructWithOptList STRUCT_PERSON_HITHCOCK_LIST = {"Alfred", "Hithcock", 81, {}};
+ static const NUdf::PersonStructWithOptList STRUCT_PERSON_LOVECRAFT_LIST = {"Howard", "Lovecraft", 25, {3, 2, 99}};
+ static const NUdf::PersonStructWithOptList STRUCT_PERSON_KING_LIST = {"Stephen", "King", 25, {}};
+
+ SIMPLE_UDF_RUN(TPersonStruct, NUdf::PersonStruct(ui32), NUdf::TOptional<void>) {
+ Y_UNUSED(valueBuilder);
+ const ui32 holePos = args[0].Get<ui32>();
+ NUdf::IBoxedValuePtr boxed(new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_JONNIE, holePos));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ typedef NUdf::TTuple<NUdf::PersonStructWithOptList,NUdf::PersonStruct,NUdf::PersonStructWithOptList,NUdf::PersonStruct> NUdfPersonTuple;
+ static const auto TUPLE_OF_PERSON = std::make_tuple(
+ STRUCT_PERSON_HITHCOCK_LIST,
+ STRUCT_PERSON_JONNIE,
+ STRUCT_PERSON_LOVECRAFT_LIST,
+ STRUCT_PERSON_KING);
+
+ static const auto TUPLE_OF_PERSON_NO_LIST = std::make_tuple(STRUCT_PERSON_HITHCOCK_LIST,
+ STRUCT_PERSON_JONNIE,
+ STRUCT_PERSON_KING_LIST,
+ STRUCT_PERSON_KING);
+
+ SIMPLE_UDF(TTupleOfPersonStruct, NUdfPersonTuple(ui32)) {
+ Y_UNUSED(valueBuilder);
+ const ui32 holePos = args[0].Get<ui32>();
+ NUdf::IBoxedValuePtr boxed(new TBrokenTupleBoxedValue<decltype(TUPLE_OF_PERSON)>(TUPLE_OF_PERSON, holePos));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ SIMPLE_UDF(TTupleOfPersonStructNoList, NUdfPersonTuple(ui32)) {
+ Y_UNUSED(valueBuilder);
+ const ui32 holePos = args[0].Get<ui32>();
+ NUdf::IBoxedValuePtr boxed(new TBrokenTupleBoxedValue<decltype(TUPLE_OF_PERSON_NO_LIST)>(TUPLE_OF_PERSON_NO_LIST, holePos));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ static const std::vector<NUdf::PersonStructWithOptList> LIST_OF_STRUCT_PERSON = {
+ STRUCT_PERSON_HITHCOCK_LIST,
+ STRUCT_PERSON_LOVECRAFT_LIST,
+ STRUCT_PERSON_KING_LIST
+ };
+
+ typedef NUdf::TDict<ui64,NUdf::PersonStructWithOptList> TIndexDictFromPersonList;
+ SIMPLE_UDF(TListOfPersonStructToIndexDict, TIndexDictFromPersonList(ui32)) {
+ Y_UNUSED(valueBuilder);
+ Y_UNUSED(args);
+ NUdf::IBoxedValuePtr boxed(new NUdf::TListRef<decltype(LIST_OF_STRUCT_PERSON)>(LIST_OF_STRUCT_PERSON));
+ return valueBuilder->ToIndexDict(NUdf::TUnboxedValuePod(std::move(boxed)));
+ }
+
+ SIMPLE_UDF(TListOfPersonStruct, NUdf::TListType<NUdf::PersonStructWithOptList>(ui32)) {
+ Y_UNUSED(valueBuilder);
+ Y_UNUSED(args);
+ NUdf::IBoxedValuePtr boxed(new NUdf::TListRef<decltype(LIST_OF_STRUCT_PERSON)>(LIST_OF_STRUCT_PERSON));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ SIMPLE_UDF(TListOfPersonStructWithBrokenIndexToDict, NUdf::TListType<NUdf::PersonStructWithOptList>()) {
+ Y_UNUSED(valueBuilder);
+ Y_UNUSED(args);
+ NUdf::IBoxedValuePtr boxed(new NUdf::TListRef<decltype(LIST_OF_STRUCT_PERSON), RAW_BROKEN_INDEX_LIST_TO_DICT>(
+ LIST_OF_STRUCT_PERSON));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ static const NUdf::PersonStruct* DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[] = {
+ &STRUCT_PERSON_HITHCOCK, &STRUCT_PERSON_JONNIE, &STRUCT_PERSON_LOVECRAFT
+ };
+ const ui32 DICT_DIGIT2PERSON_BROKEN_PERSON_INDEX = 1;
+ const ui32 DICT_DIGIT2PERSON_BROKEN_STRUCT_INDEX = 2;
+
+ const std::vector<std::pair<ui32, NUdf::IBoxedValuePtr>> MAKE_DICT_DIGIT2PERSON_BROKEN() {
+ std::vector<std::pair<ui32, NUdf::IBoxedValuePtr>> DICT_DIGIT2PERSON_BROKEN = {
+ { 333, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_HITHCOCK, RAW_INDEX_NO_HOLE) },
+ { 5, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_JONNIE, DICT_DIGIT2PERSON_BROKEN_STRUCT_INDEX) },
+ { 77, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_LOVECRAFT, RAW_INDEX_NO_HOLE) },
+ };
+
+ return DICT_DIGIT2PERSON_BROKEN;
+ }
+
+ typedef NUdf::TDict<ui32,NUdf::PersonStruct> NUdfDictDigPerson;
+
+ std::vector<std::pair<ui32, NUdf::IBoxedValuePtr>> MAKE_DICT_DIGIT2PERSON() {
+ const std::vector<std::pair<ui32, NUdf::IBoxedValuePtr>> DICT_DIGIT2PERSON = {
+ { 333, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_HITHCOCK, RAW_INDEX_NO_HOLE) },
+ { 5, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_JONNIE, RAW_INDEX_NO_HOLE) },
+ { 77, new TBrokenStructBoxedValue<NUdf::PersonStruct>(STRUCT_PERSON_LOVECRAFT, RAW_INDEX_NO_HOLE) },
+ };
+
+ return DICT_DIGIT2PERSON;
+ }
+
+ SIMPLE_UDF(TDictOfPerson, NUdfDictDigPerson()) {
+ Y_UNUSED(args);
+ Y_UNUSED(valueBuilder);
+ NUdf::IBoxedValuePtr boxed(new TBrokenDictBoxedValue<ui32, NUdf::IBoxedValuePtr>(
+ MAKE_DICT_DIGIT2PERSON(), std::make_pair(RAW_INDEX_NO_HOLE, RAW_INDEX_NO_HOLE)));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ SIMPLE_UDF_RUN(TDictOfPersonBroken, NUdfDictDigPerson(), NUdf::TOptional<void>) {
+ Y_UNUSED(args);
+ Y_UNUSED(valueBuilder);
+ NUdf::IBoxedValuePtr boxed(new TBrokenDictBoxedValue<ui32, NUdf::IBoxedValuePtr>(
+ MAKE_DICT_DIGIT2PERSON_BROKEN(), std::make_pair(RAW_INDEX_NO_HOLE, RAW_INDEX_NO_HOLE)));
+ return NUdf::TUnboxedValuePod(std::move(boxed));
+ }
+
+ SIMPLE_MODULE(TUtUDF,
+ TException,
+ TVoid,
+ TNonEmpty,
+ TOptionalNonEmpty,
+ TOptionalEmpty,
+ TSub2Mul2BrokenOnLess2,
+ TBackSub2Mul2,
+ TSeqList,
+ TSeqListWithHole,
+ TTuple,
+ TDictDigDig,
+ TDictDigDigHoleAsOpt,
+ TPersonStruct,
+ TTupleOfPersonStruct,
+ TTupleOfPersonStructNoList,
+ TListOfPersonStructToIndexDict,
+ TListOfPersonStruct,
+ TListOfPersonStructWithBrokenIndexToDict,
+ TDictOfPerson,
+ TDictOfPersonBroken
+ )
+} // unnamed namespace
+
+TIntrusivePtr<IFunctionRegistry> CreateFunctionRegistryWithUDFs() {
+ auto freg = CreateFunctionRegistry(CreateBuiltinRegistry())->Clone();
+ freg->AddModule("", "UtUDF", new TUtUDF());
+ return freg;
+}
+
+Y_UNIT_TEST_SUITE(TMiniKQLValidateTest) {
+ typedef std::function<std::vector<TRuntimeNode>(TProgramBuilder&)> BuildArgsFunc;
+ typedef std::function<void(const NUdf::TUnboxedValuePod&, const NUdf::IValueBuilder*)> ValidateValueFunc;
+ typedef std::function<void(const NUdf::TUnboxedValuePod&, const NUdf::IValueBuilder*, const TType* type)> FullValidateValueFunc;
+
+
+ void ProcessSimpleUdfFunc(const char* udfFuncName, BuildArgsFunc argsFunc = BuildArgsFunc(), ValidateValueFunc validateFunc = ValidateValueFunc(),
+ FullValidateValueFunc fullValidateFunc = FullValidateValueFunc(),
+ NUdf::EValidateMode validateMode = NUdf::EValidateMode::Lazy) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper);
+ auto functionRegistry = CreateFunctionRegistryWithUDFs();
+ auto randomProvider = CreateDeterministicRandomProvider(1);
+ auto timeProvider = CreateDeterministicTimeProvider(10000000);
+ TProgramBuilder pgmBuilder(env, *functionRegistry);
+
+ auto funcName = pgmBuilder.Udf(udfFuncName);
+ std::vector<TRuntimeNode> execArgs;
+ if (argsFunc) {
+ execArgs = argsFunc(pgmBuilder);
+ }
+ auto pgmReturn = pgmBuilder.Apply(funcName, execArgs);
+
+ TExploringNodeVisitor explorer;
+ explorer.Walk(pgmReturn.GetNode(), env);
+ TComputationPatternOpts opts(alloc.Ref(), env, GetBuiltinFactory(),
+ functionRegistry.Get(), validateMode,
+ NUdf::EValidatePolicy::Exception, "OFF", EGraphPerProcess::Multi);
+ auto pattern = MakeComputationPattern(explorer, pgmReturn, {}, opts);
+ auto graph = pattern->Clone(opts.ToComputationOptions(*randomProvider, *timeProvider));
+ const auto builder = static_cast<TDefaultValueBuilder*>(graph->GetTerminator());
+ builder->RethrowAtTerminate();
+ const TBindTerminator bind(graph->GetTerminator());
+ auto value = graph->GetValue();
+
+ if (validateFunc) {
+ validateFunc(value, builder);
+ }
+ if (fullValidateFunc) {
+ ui32 flags = 0;
+ TFunctionTypeInfo funcInfo;
+ TType* userType = nullptr;
+ TStringBuf typeConfig;
+ TStatus status = functionRegistry->FindFunctionTypeInfo(env, typeInfoHelper, nullptr, udfFuncName, userType, typeConfig, flags, {}, nullptr, &funcInfo);
+ MKQL_ENSURE(status.IsOk(), status.GetError());
+ auto type = funcInfo.FunctionType->GetReturnType();
+ fullValidateFunc(value, builder, type);
+ }
+ }
+
+ Y_UNIT_TEST(TestUdfException) {
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ valueBuilder->NewStringNotFilled(0xBAD).AsStringValue().Ref(); // Leak string.
+ NUdf::TBoxedValueAccessor::Skip(*value.GetListIterator().AsBoxed().Release()); // Leak value and throw exception.
+ };
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.Exception", {}, validateFunc), yexception);
+ UNIT_ASSERT_VALUES_EQUAL(TThrowerValue::Count, 0L);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckVoid) {
+ ProcessSimpleUdfFunc("UtUDF.Void");
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckExceptionOnEmpty) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ bool wrapped = false;
+ UNIT_ASSERT_EXCEPTION(TValidate<TValidateErrorPolicyThrow>::Value(nullptr, env.GetTypeOfType(),
+ NUdf::TUnboxedValuePod(), "ut for verify empty value exception", &wrapped), TUdfValidateException);
+ UNIT_ASSERT(!wrapped);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckNonEmpty) {
+ ProcessSimpleUdfFunc("UtUDF.NonEmpty");
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckOptionalNonEmpty) {
+ ProcessSimpleUdfFunc("UtUDF.OptionalNonEmpty");
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckOptionalEmpty) {
+ ProcessSimpleUdfFunc("UtUDF.OptionalEmpty");
+ }
+
+ std::vector<TRuntimeNode> MakeCallableInArgs(ui32 testVal, TProgramBuilder& pgmBuilder) {
+ const auto& functionRegistry = pgmBuilder.GetFunctionRegistry();
+
+ const auto udfFuncName = "UtUDF.Sub2Mul2BrokenOnLess2";
+ ui32 flags = 0;
+ TFunctionTypeInfo funcInfo;
+ TType* userType = nullptr;
+ TStringBuf typeConfig;
+ NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper);
+ TStatus status = functionRegistry.FindFunctionTypeInfo(pgmBuilder.GetTypeEnvironment(), typeInfoHelper, nullptr,
+ udfFuncName, userType, typeConfig, flags, {}, nullptr, &funcInfo);
+ MKQL_ENSURE(status.IsOk(), status.GetError());
+ auto callable = pgmBuilder.Udf(udfFuncName);
+ return std::vector<TRuntimeNode>{callable, pgmBuilder.NewDataLiteral(testVal)};
+ };
+
+ Y_UNIT_TEST(TestVerifyArgsCallableCorrect) {
+ ui32 testVal = 44;
+ BuildArgsFunc argsFunc = [testVal](TProgramBuilder& pgmBuilder) {
+ return MakeCallableInArgs(testVal, pgmBuilder);
+ };
+ ValidateValueFunc validateFunc = [testVal](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ UNIT_ASSERT_VALUES_EQUAL(testVal, value.Get<ui32>());
+ };
+ ProcessSimpleUdfFunc("UtUDF.BackSub2Mul2", argsFunc, validateFunc);
+ }
+
+ Y_UNIT_TEST(TestVerifyArgsCallableBrokenOnArgument) {
+ ui32 testVal = 101;
+ BuildArgsFunc argsFunc = [testVal](TProgramBuilder& pgmBuilder) {
+ return MakeCallableInArgs(testVal, pgmBuilder);
+ };
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.BackSub2Mul2", argsFunc), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestVerifyArgsCallableBrokenOnReturn) {
+ ui32 testVal = 1;
+ BuildArgsFunc argsFunc = [testVal](TProgramBuilder& pgmBuilder) {
+ return MakeCallableInArgs(testVal, pgmBuilder);
+ };
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.BackSub2Mul2", argsFunc), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckEmptySeqList) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(0)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ auto listIter = value.GetListIterator();
+ UNIT_ASSERT(!listIter.Skip());
+ };
+ ProcessSimpleUdfFunc("UtUDF.SeqList", argsFunc, validateFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckSeqList) {
+ static constexpr ui32 listSize = 31;
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(listSize)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ ui32 index = 0;
+ auto listIter = value.GetListIterator();
+ for (NUdf::TUnboxedValue item; listIter.Next(item); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), index);
+ }
+ UNIT_ASSERT_VALUES_EQUAL(index, listSize);
+ };
+ ProcessSimpleUdfFunc("UtUDF.SeqList", argsFunc, validateFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckSeqListWithHoleFirst) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ const ui32 listSize = 31;
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(listSize),
+ pgmBuilder.NewDataLiteral<ui32>(0)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ auto listIter = value.GetListIterator();
+ NUdf::TUnboxedValue item;
+ UNIT_ASSERT_EXCEPTION(listIter.Next(item), TUdfValidateException);
+ for (ui32 index = 1; listIter.Next(item); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), index);
+ }
+ };
+ ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc);
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc, {}, NUdf::EValidateMode::Greedy), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckSeqListWithHoleMiddle) {
+ static constexpr ui32 listSize = 31;
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(listSize),
+ pgmBuilder.NewDataLiteral(listSize / 2)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), listSize);
+ ui32 index = 0;
+ const auto listIter = value.GetListIterator();
+ for (NUdf::TUnboxedValue item; index < listSize / 2 && listIter.Next(item); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), index);
+ }
+ NUdf::TUnboxedValue bad;
+ UNIT_ASSERT_EXCEPTION(listIter.Next(bad), TUdfValidateException);
+ ++index;
+ for (NUdf::TUnboxedValue item; listIter.Next(item); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), index);
+ }
+ };
+ ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc);
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc, {}, NUdf::EValidateMode::Greedy), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckSeqListWithHoleLast) {
+ static constexpr ui32 listSize = 31;
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(listSize),
+ pgmBuilder.NewDataLiteral(listSize - 1)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ UNIT_ASSERT_VALUES_EQUAL(value.GetListLength(), listSize);
+ ui32 index = 0;
+ auto listIter = value.GetListIterator();
+ for (NUdf::TUnboxedValue item; index < listSize - 1 && listIter.Next(item); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), index);
+ }
+ UNIT_ASSERT_VALUES_EQUAL(index, listSize - 1);
+ NUdf::TUnboxedValue bad;
+ UNIT_ASSERT_EXCEPTION(listIter.Next(bad), TUdfValidateException);
+ };
+ ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc);
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.SeqListWithHole", argsFunc, validateFunc, {}, NUdf::EValidateMode::Greedy), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckTuple) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)};
+ };
+ ProcessSimpleUdfFunc("UtUDF.Tuple", argsFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckTupleWithHoleFirst) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(0)};
+ };
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.Tuple", argsFunc), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckTupleWithHoleMiddle) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(std::tuple_size<decltype(TUPLE)>::value / 2)};
+ };
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.Tuple", argsFunc), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckTupleWithHoleLast) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(std::tuple_size<decltype(TUPLE)>::value - 1)};
+ };
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.Tuple", argsFunc), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckDictDigitDigitFull) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE),
+ pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ auto dictIter = value.GetDictIterator();
+ ui32 index = 0;
+ for (NUdf::TUnboxedValue key, payload; dictIter.NextPair(key, payload); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), DICT_DIGIT2DIGIT[index].first);
+ UNIT_ASSERT_VALUES_EQUAL(payload.Get<ui64>(), DICT_DIGIT2DIGIT[index].second);
+ }
+ UNIT_ASSERT_VALUES_EQUAL(index, DICT_DIGIT2DIGIT.size());
+ };
+ ProcessSimpleUdfFunc("UtUDF.DictDigDig", argsFunc, validateFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckDictDigitDigitKeyHole) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(0),
+ pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ auto dictIter = value.GetDictIterator();
+ NUdf::TUnboxedValue key, payload;
+ UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException);
+ for (ui32 index = 1; dictIter.NextPair(key, payload); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), DICT_DIGIT2DIGIT[index].first);
+ UNIT_ASSERT_VALUES_EQUAL(payload.Get<ui64>(), DICT_DIGIT2DIGIT[index].second);
+ }
+ };
+ ProcessSimpleUdfFunc("UtUDF.DictDigDig", argsFunc, validateFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckDictDigitDigitValueHole) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE),
+ pgmBuilder.NewDataLiteral<ui32>(DICT_DIGIT2DIGIT.size() - 1)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ auto dictIter = value.GetDictIterator();
+ NUdf::TUnboxedValue key, payload;
+ for (ui32 index = 0; index < DICT_DIGIT2DIGIT.size() - 1 && dictIter.NextPair(key, payload); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), DICT_DIGIT2DIGIT[index].first);
+ UNIT_ASSERT_VALUES_EQUAL(payload.Get<ui64>(), DICT_DIGIT2DIGIT[index].second);
+ }
+ UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException);
+ };
+ ProcessSimpleUdfFunc("UtUDF.DictDigDig", argsFunc, validateFunc);
+ }
+ Y_UNIT_TEST(TestUdfResultCheckDictDigitDigitHoleAsOptKeyHole) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE),
+ pgmBuilder.NewDataLiteral<ui32>(0)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ auto dictIter = value.GetDictIterator();
+ NUdf::TUnboxedValue key, payload;
+ UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException);
+ for (ui32 index = 1; dictIter.NextPair(key, payload); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), DICT_DIGIT2DIGIT[index].first);
+ UNIT_ASSERT_VALUES_EQUAL(payload.Get<ui64>(), DICT_DIGIT2DIGIT[index].second);
+ }
+ };
+ ProcessSimpleUdfFunc("UtUDF.DictDigDig", argsFunc, validateFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckDictDigitDigitHoleAsOptValueHole) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(DICT_DIGIT2DIGIT.size() - 1),
+ pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ auto dictIter = value.GetDictIterator();
+ NUdf::TUnboxedValue key, payload;
+ for (ui32 index = 0; index < DICT_DIGIT2DIGIT.size() - 1 && dictIter.NextPair(key, payload); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), DICT_DIGIT2DIGIT[index].first);
+ UNIT_ASSERT_VALUES_EQUAL(payload.Get<ui64>(), DICT_DIGIT2DIGIT[index].second);
+ }
+ UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException);
+ };
+ ProcessSimpleUdfFunc("UtUDF.DictDigDig", argsFunc, validateFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckPersonStruct) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)};
+ };
+ ProcessSimpleUdfFunc("UtUDF.PersonStruct", argsFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckPersonStructWithHoleFirst) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(0)};
+ };
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.PersonStruct", argsFunc), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckPersonStructWithHoleMiddle) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(1)};
+ };
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.PersonStruct", argsFunc), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckPersonStructWithHoleLast) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral<ui32>(2)};
+ };
+ UNIT_ASSERT_EXCEPTION(ProcessSimpleUdfFunc("UtUDF.PersonStruct", argsFunc), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckTupleOfPersonStruct) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)};
+ };
+ FullValidateValueFunc fullValidateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder,
+ const TType* type) {
+ bool wrapped = false;
+ TValidate<TValidateErrorPolicyThrow, TValidateModeGreedy<TValidateErrorPolicyThrow>>::Value(valueBuilder, type, NUdf::TUnboxedValuePod(value), "full verify func", &wrapped);
+ UNIT_ASSERT(!wrapped);
+ TValidate<TValidateErrorPolicyThrow, TValidateModeLazy<TValidateErrorPolicyThrow>>::Value(valueBuilder, type, NUdf::TUnboxedValuePod(value), "full verify func", &wrapped);
+ UNIT_ASSERT(wrapped);
+ };
+ ProcessSimpleUdfFunc("UtUDF.TupleOfPersonStruct", argsFunc, {}, fullValidateFunc);
+ ProcessSimpleUdfFunc("UtUDF.TupleOfPersonStruct", argsFunc, {}, fullValidateFunc, NUdf::EValidateMode::Greedy);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckTupleOfPersonStructNoList) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)};
+ };
+ FullValidateValueFunc fullValidateFunc = [](const NUdf::TUnboxedValuePod& value,
+ const NUdf::IValueBuilder* valueBuilder, const TType* type) {
+ bool wrapped = false;
+ TValidate<TValidateErrorPolicyThrow>::Value(valueBuilder, type, NUdf::TUnboxedValuePod(value), "full verify func", &wrapped);
+ UNIT_ASSERT(!wrapped);
+ };
+ ProcessSimpleUdfFunc("UtUDF.TupleOfPersonStructNoList", argsFunc, {}, fullValidateFunc);
+ ProcessSimpleUdfFunc("UtUDF.TupleOfPersonStructNoList", argsFunc, {}, fullValidateFunc, NUdf::EValidateMode::Greedy);
+ }
+
+ void ValidateDictOfPersonStructFunc(const NUdf::TUnboxedValuePod& value, ui32 lookupIndex = 2, ui32 broken_index = RAW_INDEX_NO_HOLE) {
+ const auto person = value.Lookup(NUdf::TUnboxedValuePod(ui64(lookupIndex)));
+ UNIT_ASSERT(person);
+ auto firstName = person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[0]);
+ UNIT_ASSERT_VALUES_EQUAL(TString(firstName.AsStringRef()), LIST_OF_STRUCT_PERSON[lookupIndex].FirstName);
+ auto lastName = person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[1]);
+ UNIT_ASSERT_VALUES_EQUAL(TString(lastName.AsStringRef()), LIST_OF_STRUCT_PERSON[lookupIndex].LastName);
+ UNIT_ASSERT_VALUES_EQUAL(person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[2]).Get<ui32>(), LIST_OF_STRUCT_PERSON[lookupIndex].Age);
+ UNIT_ASSERT(!person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[3]));
+ auto dictIter = value.GetDictIterator();
+ NUdf::TUnboxedValue key, payload;
+ for (ui32 index = 0; index < broken_index && dictIter.NextPair(key, payload); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(key.Get<ui64>(), index);
+ auto person = payload;
+ auto firstName = person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[0]);
+ UNIT_ASSERT_VALUES_EQUAL(TString(firstName.AsStringRef()), LIST_OF_STRUCT_PERSON[index].FirstName);
+ auto lastName = person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[1]);
+ UNIT_ASSERT_VALUES_EQUAL(TString(lastName.AsStringRef()), LIST_OF_STRUCT_PERSON[index].LastName);
+ UNIT_ASSERT_VALUES_EQUAL(person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[2]).Get<ui32>(), LIST_OF_STRUCT_PERSON[index].Age);
+ const auto origListPtr = LIST_OF_STRUCT_PERSON[index].Tags;
+ if (origListPtr.empty()) {
+ UNIT_ASSERT(!person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[3]));
+ } else {
+ auto memberTags = person.GetElement(NUdf::PersonStructWithOptList::MetaIndexes[3]);
+ UNIT_ASSERT(memberTags);
+ UNIT_ASSERT_VALUES_EQUAL(memberTags.GetListLength(), origListPtr.size());
+ auto origIter = origListPtr.begin();
+ auto iter = memberTags.GetListIterator();
+ for (NUdf::TUnboxedValue item; iter.Next(item); ++origIter) {
+ UNIT_ASSERT_VALUES_EQUAL(item.Get<ui32>(), *origIter);
+ }
+ }
+ }
+ if (broken_index < RAW_INDEX_NO_HOLE)
+ UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckListOfPersonStructToIndexDict) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)};
+ };
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder) {
+ Y_UNUSED(valueBuilder);
+ ValidateDictOfPersonStructFunc(value);
+ };
+ ProcessSimpleUdfFunc("UtUDF.ListOfPersonStructToIndexDict", argsFunc, validateFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckListOfPersonStruct) {
+ BuildArgsFunc argsFunc = [](TProgramBuilder& pgmBuilder) {
+ return std::vector<TRuntimeNode>{pgmBuilder.NewDataLiteral(RAW_INDEX_NO_HOLE)};
+ };
+ FullValidateValueFunc fullValidateFunc = [](const NUdf::TUnboxedValuePod& value,
+ const NUdf::IValueBuilder* valueBuilder, const TType* type) {
+ Y_UNUSED(type);
+ auto indexDict = valueBuilder->ToIndexDict(value);
+ ValidateDictOfPersonStructFunc(indexDict);
+ };
+ ProcessSimpleUdfFunc("UtUDF.ListOfPersonStruct", argsFunc, {}, fullValidateFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckListOfPersonStructWithBrokenIndexToDict) {
+ FullValidateValueFunc fullValidateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder* valueBuilder,
+ const TType* type) {
+ Y_UNUSED(type);
+ auto indexDict = valueBuilder->ToIndexDict(value);
+ static_assert(RAW_BROKEN_INDEX_LIST_TO_DICT == 1, "a list is too small");
+ ValidateDictOfPersonStructFunc(indexDict, RAW_BROKEN_INDEX_LIST_TO_DICT - 1, RAW_BROKEN_INDEX_LIST_TO_DICT);
+ /// verify lookup fail on broken index
+ UNIT_ASSERT_EXCEPTION(ValidateDictOfPersonStructFunc(indexDict, RAW_BROKEN_INDEX_LIST_TO_DICT, RAW_BROKEN_INDEX_LIST_TO_DICT), TUdfValidateException);
+ ValidateDictOfPersonStructFunc(indexDict, RAW_BROKEN_INDEX_LIST_TO_DICT + 1, RAW_BROKEN_INDEX_LIST_TO_DICT);
+ };
+ ProcessSimpleUdfFunc("UtUDF.ListOfPersonStructWithBrokenIndexToDict", {}, {}, fullValidateFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckDictOfPerson) {
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder*) {
+ auto dictIter = value.GetDictIterator();
+ NUdf::TUnboxedValue key, payload;
+ for (ui32 index = 0; dictIter.NextPair(key, payload); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), MAKE_DICT_DIGIT2PERSON()[index].first);
+ auto person = payload;
+ auto firstName = person.GetElement(NUdf::PersonStruct::MetaIndexes[0]);
+ UNIT_ASSERT_VALUES_EQUAL(TString(firstName.AsStringRef()), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->FirstName);
+ auto lastName = person.GetElement(NUdf::PersonStruct::MetaIndexes[1]);
+ UNIT_ASSERT_VALUES_EQUAL(TString(lastName.AsStringRef()), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->LastName);
+ UNIT_ASSERT_VALUES_EQUAL(person.GetElement(NUdf::PersonStruct::MetaIndexes[2]).Get<ui32>(), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->Age);
+ }
+ };
+ ProcessSimpleUdfFunc("UtUDF.DictOfPerson", {}, validateFunc);
+ }
+
+ Y_UNIT_TEST(TestUdfResultCheckDictOfPersonBroken) {
+ ValidateValueFunc validateFunc = [](const NUdf::TUnboxedValuePod& value, const NUdf::IValueBuilder*) {
+ auto dictIter = value.GetDictIterator();
+ NUdf::TUnboxedValue key, payload;
+ for (ui32 index = 0; index < DICT_DIGIT2PERSON_BROKEN_PERSON_INDEX && dictIter.NextPair(key, payload); ++index) {
+ UNIT_ASSERT_VALUES_EQUAL(key.Get<ui32>(), MAKE_DICT_DIGIT2PERSON_BROKEN()[index].first);
+ auto person = payload;
+ auto firstName = person.GetElement(NUdf::PersonStruct::MetaIndexes[0]);
+ UNIT_ASSERT_VALUES_EQUAL(TString(firstName.AsStringRef()), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->FirstName);
+ auto lastName = person.GetElement(NUdf::PersonStruct::MetaIndexes[1]);
+ UNIT_ASSERT_VALUES_EQUAL(TString(lastName.AsStringRef()), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->LastName);
+ UNIT_ASSERT_VALUES_EQUAL(person.GetElement(NUdf::PersonStruct::MetaIndexes[2]).Get<ui32>(), DICT_DIGIT2PERSON_BROKEN_CONTENT_BY_INDEX[index]->Age);
+ }
+ UNIT_ASSERT_EXCEPTION(dictIter.NextPair(key, payload), TUdfValidateException);
+ };
+ ProcessSimpleUdfFunc("UtUDF.DictOfPersonBroken", {}, validateFunc);
+ }
+
+}
+
+}
diff --git a/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp b/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp
new file mode 100644
index 00000000000..46a04cc001c
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp
@@ -0,0 +1,373 @@
+#include "mkql_value_builder.h"
+#include "mkql_computation_node_holders.h"
+
+#include <ydb/library/yql/minikql/mkql_function_registry.h>
+#include <ydb/library/yql/minikql/mkql_type_builder.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/parser/pg_catalog/catalog.h>
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <arrow/array/builder_primitive.h>
+#include <arrow/c/abi.h>
+#include <arrow/scalar.h>
+#include <arrow/chunked_array.h>
+
+namespace NYql {
+namespace NCommon {
+
+TString PgValueToNativeText(const NUdf::TUnboxedValuePod& value, ui32 pgTypeId);
+TString PgValueToNativeBinary(const NUdf::TUnboxedValuePod& value, ui32 pgTypeId);
+
+}
+}
+
+namespace NKikimr {
+
+using namespace NUdf;
+using namespace NYql::NCommon;
+
+namespace NMiniKQL {
+
+namespace {
+ TString AsString(const TStringValue& v) {
+ return { v.Data(), v.Size() };
+ }
+}
+
+class TMiniKQLValueBuilderTest: public TTestBase {
+public:
+ TMiniKQLValueBuilderTest()
+ : FunctionRegistry(CreateFunctionRegistry(CreateBuiltinRegistry()))
+ , Alloc(__LOCATION__)
+ , Env(Alloc)
+ , MemInfo("Memory")
+ , HolderFactory(Alloc.Ref(), MemInfo, FunctionRegistry.Get())
+ , Builder(HolderFactory, NUdf::EValidatePolicy::Exception)
+ , TypeInfoHelper(new TTypeInfoHelper())
+ , FunctionTypeInfoBuilder(Env, TypeInfoHelper, "", nullptr, {})
+ {
+ BoolOid = NYql::NPg::LookupType("bool").TypeId;
+ }
+
+ const IPgBuilder& GetPgBuilder() const {
+ return Builder.GetPgBuilder();
+ }
+
+private:
+ TIntrusivePtr<NMiniKQL::IFunctionRegistry> FunctionRegistry;
+ TScopedAlloc Alloc;
+ TTypeEnvironment Env;
+ TMemoryUsageInfo MemInfo;
+ THolderFactory HolderFactory;
+ TDefaultValueBuilder Builder;
+ NUdf::ITypeInfoHelper::TPtr TypeInfoHelper;
+ TFunctionTypeInfoBuilder FunctionTypeInfoBuilder;
+ ui32 BoolOid = 0;
+
+ UNIT_TEST_SUITE(TMiniKQLValueBuilderTest);
+ UNIT_TEST(TestEmbeddedVariant);
+ UNIT_TEST(TestBoxedVariant);
+ UNIT_TEST(TestSubstring);
+ UNIT_TEST(TestPgValueFromErrors);
+ UNIT_TEST(TestPgValueFromText);
+ UNIT_TEST(TestPgValueFromBinary);
+ UNIT_TEST(TestConvertToFromPg);
+ UNIT_TEST(TestConvertToFromPgNulls);
+ UNIT_TEST(TestPgNewString);
+ UNIT_TEST(TestArrowBlock);
+ UNIT_TEST_SUITE_END();
+
+
+ void TestEmbeddedVariant() {
+ const auto v = Builder.NewVariant(62, TUnboxedValuePod((ui64) 42));
+ UNIT_ASSERT(v);
+ UNIT_ASSERT(!v.IsBoxed());
+ UNIT_ASSERT_VALUES_EQUAL(62, v.GetVariantIndex());
+ UNIT_ASSERT_VALUES_EQUAL(42, v.GetVariantItem().Get<ui64>());
+ }
+
+ void TestBoxedVariant() {
+ const auto v = Builder.NewVariant(63, TUnboxedValuePod((ui64) 42));
+ UNIT_ASSERT(v);
+ UNIT_ASSERT(v.IsBoxed());
+ UNIT_ASSERT_VALUES_EQUAL(63, v.GetVariantIndex());
+ UNIT_ASSERT_VALUES_EQUAL(42, v.GetVariantItem().Get<ui64>());
+ }
+
+ void TestSubstring() {
+ const auto string = Builder.NewString("0123456789qwertyuiopasdfghjklzxcvbnmQWERTYUIOPASDFGHJKLZXCVBNM");
+ UNIT_ASSERT(string);
+
+ const auto zero = Builder.SubString(string, 7, 0);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(""), TStringBuf(zero.AsStringRef()));
+
+ const auto tail = Builder.SubString(string, 60, 8);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf("NM"), TStringBuf(tail.AsStringRef()));
+
+ const auto small = Builder.SubString(string, 2, 14);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf("23456789qwerty"), TStringBuf(small.AsStringRef()));
+
+ const auto one = Builder.SubString(string, 3, 15);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf("3456789qwertyui"), TStringBuf(one.AsStringRef()));
+ UNIT_ASSERT_VALUES_EQUAL(string.AsStringValue().Data(), one.AsStringValue().Data());
+
+ const auto two = Builder.SubString(string, 10, 30);
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf("qwertyuiopasdfghjklzxcvbnmQWER"), TStringBuf(two.AsStringRef()));
+ UNIT_ASSERT_VALUES_EQUAL(string.AsStringValue().Data(), two.AsStringValue().Data());
+ }
+
+ void TestPgValueFromErrors() {
+ const TBindTerminator bind(&Builder); // to raise exception instead of abort
+ {
+ TStringValue error("");
+ auto r = GetPgBuilder().ValueFromText(BoolOid, "", error);
+ UNIT_ASSERT(!r);
+ UNIT_ASSERT_STRING_CONTAINS(AsString(error), "ERROR: invalid input syntax for type boolean: \"\"");
+ }
+
+ {
+ TStringValue error("");
+ auto r = GetPgBuilder().ValueFromText(BoolOid, "zzzz", error);
+ UNIT_ASSERT(!r);
+ UNIT_ASSERT_STRING_CONTAINS(AsString(error), "ERROR: invalid input syntax for type boolean: \"zzzz\"");
+ }
+
+ {
+ TStringValue error("");
+ auto r = GetPgBuilder().ValueFromBinary(BoolOid, "", error);
+ UNIT_ASSERT(!r);
+ UNIT_ASSERT_STRING_CONTAINS(AsString(error), "ERROR: no data left in message");
+ }
+
+ {
+ TStringValue error("");
+ auto r = GetPgBuilder().ValueFromBinary(BoolOid, "zzzz", error);
+ UNIT_ASSERT(!r);
+ UNIT_ASSERT_STRING_CONTAINS(AsString(error), "Not all data has been consumed by 'recv' function: boolrecv, data size: 4, consumed size: 1");
+ }
+ }
+
+ void TestPgValueFromText() {
+ const TBindTerminator bind(&Builder);
+ for (auto validTrue : { "t"sv, "true"sv }) {
+ TStringValue error("");
+ auto r = GetPgBuilder().ValueFromText(BoolOid, validTrue, error);
+ UNIT_ASSERT(r);
+ UNIT_ASSERT_VALUES_EQUAL(AsString(error), "");
+ auto s = PgValueToNativeText(r, BoolOid);
+ UNIT_ASSERT_VALUES_EQUAL(s, "t");
+ }
+
+ for (auto validFalse : { "f"sv, "false"sv }) {
+ TStringValue error("");
+ auto r = GetPgBuilder().ValueFromText(BoolOid, validFalse, error);
+ UNIT_ASSERT(r);
+ UNIT_ASSERT_VALUES_EQUAL(AsString(error), "");
+ auto s = PgValueToNativeText(r, BoolOid);
+ UNIT_ASSERT_VALUES_EQUAL(s, "f");
+ }
+ }
+
+ void TestPgValueFromBinary() {
+ const TBindTerminator bind(&Builder);
+ TStringValue error("");
+ auto t = GetPgBuilder().ValueFromText(BoolOid, "true", error);
+ UNIT_ASSERT(t);
+ auto f = GetPgBuilder().ValueFromText(BoolOid, "false", error);
+ UNIT_ASSERT(f);
+
+ auto ts = PgValueToNativeBinary(t, BoolOid);
+ auto fs = PgValueToNativeBinary(f, BoolOid);
+ {
+ auto r = GetPgBuilder().ValueFromBinary(BoolOid, ts, error);
+ UNIT_ASSERT(r);
+ auto s = PgValueToNativeText(r, BoolOid);
+ UNIT_ASSERT_VALUES_EQUAL(s, "t");
+ }
+
+ {
+ auto r = GetPgBuilder().ValueFromBinary(BoolOid, fs, error);
+ UNIT_ASSERT(r);
+ auto s = PgValueToNativeText(r, BoolOid);
+ UNIT_ASSERT_VALUES_EQUAL(s, "f");
+ }
+ }
+
+ void TestConvertToFromPg() {
+ const TBindTerminator bind(&Builder);
+ auto boolType = FunctionTypeInfoBuilder.SimpleType<bool>();
+ {
+ auto v = GetPgBuilder().ConvertToPg(TUnboxedValuePod(true), boolType, BoolOid);
+ auto s = PgValueToNativeText(v, BoolOid);
+ UNIT_ASSERT_VALUES_EQUAL(s, "t");
+
+ auto from = GetPgBuilder().ConvertFromPg(v, BoolOid, boolType);
+ UNIT_ASSERT_VALUES_EQUAL(from.Get<bool>(), true);
+ }
+
+ {
+ auto v = GetPgBuilder().ConvertToPg(TUnboxedValuePod(false), boolType, BoolOid);
+ auto s = PgValueToNativeText(v, BoolOid);
+ UNIT_ASSERT_VALUES_EQUAL(s, "f");
+
+ auto from = GetPgBuilder().ConvertFromPg(v, BoolOid, boolType);
+ UNIT_ASSERT_VALUES_EQUAL(from.Get<bool>(), false);
+ }
+ }
+
+ void TestConvertToFromPgNulls() {
+ const TBindTerminator bind(&Builder);
+ auto boolOptionalType = FunctionTypeInfoBuilder.Optional()->Item<bool>().Build();
+
+ {
+ auto v = GetPgBuilder().ConvertToPg(TUnboxedValuePod(), boolOptionalType, BoolOid);
+ UNIT_ASSERT(!v);
+ }
+
+ {
+ auto v = GetPgBuilder().ConvertFromPg(TUnboxedValuePod(), BoolOid, boolOptionalType);
+ UNIT_ASSERT(!v);
+ }
+ }
+
+ void TestPgNewString() {
+ {
+ auto& pgText = NYql::NPg::LookupType("text");
+ UNIT_ASSERT_VALUES_EQUAL(pgText.TypeLen, -1);
+
+ auto s = GetPgBuilder().NewString(pgText.TypeLen, pgText.TypeId, "ABC");
+ auto utf8Type = FunctionTypeInfoBuilder.SimpleType<TUtf8>();
+ auto from = GetPgBuilder().ConvertFromPg(s, pgText.TypeId, utf8Type);
+ UNIT_ASSERT_VALUES_EQUAL((TStringBuf)from.AsStringRef(), "ABC"sv);
+ }
+
+ {
+ auto& pgCString = NYql::NPg::LookupType("cstring");
+ UNIT_ASSERT_VALUES_EQUAL(pgCString.TypeLen, -2);
+
+ auto s = GetPgBuilder().NewString(pgCString.TypeLen, pgCString.TypeId, "ABC");
+ auto utf8Type = FunctionTypeInfoBuilder.SimpleType<TUtf8>();
+ auto from = GetPgBuilder().ConvertFromPg(s, pgCString.TypeId, utf8Type);
+ UNIT_ASSERT_VALUES_EQUAL((TStringBuf)from.AsStringRef(), "ABC"sv);
+ }
+
+ {
+ auto& byteaString = NYql::NPg::LookupType("bytea");
+ UNIT_ASSERT_VALUES_EQUAL(byteaString.TypeLen, -1);
+
+ auto s = GetPgBuilder().NewString(byteaString.TypeLen, byteaString.TypeId, "ABC");
+ auto stringType = FunctionTypeInfoBuilder.SimpleType<char*>();
+ auto from = GetPgBuilder().ConvertFromPg(s, byteaString.TypeId, stringType);
+ UNIT_ASSERT_VALUES_EQUAL((TStringBuf)from.AsStringRef(), "ABC"sv);
+ }
+ }
+
+ void TestArrowBlock() {
+ auto type = FunctionTypeInfoBuilder.SimpleType<ui64>();
+ auto atype = TypeInfoHelper->MakeArrowType(type);
+
+ {
+ arrow::Datum d1(std::make_shared<arrow::UInt64Scalar>(123));
+ NUdf::TUnboxedValue val1 = HolderFactory.CreateArrowBlock(std::move(d1));
+ bool isScalar;
+ ui64 length;
+ auto chunks = Builder.GetArrowBlockChunks(val1, isScalar, length);
+ UNIT_ASSERT_VALUES_EQUAL(chunks, 1);
+ UNIT_ASSERT(isScalar);
+ UNIT_ASSERT_VALUES_EQUAL(length, 1);
+
+ ArrowArray arr1;
+ Builder.ExportArrowBlock(val1, 0, &arr1);
+ NUdf::TUnboxedValue val2 = Builder.ImportArrowBlock(&arr1, 1, isScalar, *atype);
+ const auto& d2 = TArrowBlock::From(val2).GetDatum();
+ UNIT_ASSERT(d2.is_scalar());
+ UNIT_ASSERT_VALUES_EQUAL(d2.scalar_as<arrow::UInt64Scalar>().value, 123);
+ }
+
+ {
+ arrow::UInt64Builder builder;
+ UNIT_ASSERT(builder.Reserve(3).ok());
+ builder.UnsafeAppend(ui64(10));
+ builder.UnsafeAppend(ui64(20));
+ builder.UnsafeAppend(ui64(30));
+ std::shared_ptr<arrow::ArrayData> builderResult;
+ UNIT_ASSERT(builder.FinishInternal(&builderResult).ok());
+ arrow::Datum d1(builderResult);
+ NUdf::TUnboxedValue val1 = HolderFactory.CreateArrowBlock(std::move(d1));
+
+ bool isScalar;
+ ui64 length;
+ auto chunks = Builder.GetArrowBlockChunks(val1, isScalar, length);
+ UNIT_ASSERT_VALUES_EQUAL(chunks, 1);
+ UNIT_ASSERT(!isScalar);
+ UNIT_ASSERT_VALUES_EQUAL(length, 3);
+
+ ArrowArray arr1;
+ Builder.ExportArrowBlock(val1, 0, &arr1);
+ NUdf::TUnboxedValue val2 = Builder.ImportArrowBlock(&arr1, 1, isScalar, *atype);
+ const auto& d2 = TArrowBlock::From(val2).GetDatum();
+ UNIT_ASSERT(d2.is_array());
+ UNIT_ASSERT_VALUES_EQUAL(d2.array()->length, 3);
+ UNIT_ASSERT_VALUES_EQUAL(d2.array()->GetNullCount(), 0);
+ auto flat = d2.array()->GetValues<ui64>(1);
+ UNIT_ASSERT_VALUES_EQUAL(flat[0], 10);
+ UNIT_ASSERT_VALUES_EQUAL(flat[1], 20);
+ UNIT_ASSERT_VALUES_EQUAL(flat[2], 30);
+ }
+
+ {
+ arrow::UInt64Builder builder1;
+ UNIT_ASSERT(builder1.Reserve(3).ok());
+ builder1.UnsafeAppend(ui64(10));
+ builder1.UnsafeAppend(ui64(20));
+ builder1.UnsafeAppend(ui64(30));
+ std::shared_ptr<arrow::Array> builder1Result;
+ UNIT_ASSERT(builder1.Finish(&builder1Result).ok());
+
+ arrow::UInt64Builder builder2;
+ UNIT_ASSERT(builder2.Reserve(2).ok());
+ builder2.UnsafeAppend(ui64(40));
+ builder2.UnsafeAppend(ui64(50));
+ std::shared_ptr<arrow::Array> builder2Result;
+ UNIT_ASSERT(builder2.Finish(&builder2Result).ok());
+
+ auto chunked = arrow::ChunkedArray::Make({ builder1Result, builder2Result }).ValueOrDie();
+ arrow::Datum d1(chunked);
+ NUdf::TUnboxedValue val1 = HolderFactory.CreateArrowBlock(std::move(d1));
+
+ bool isScalar;
+ ui64 length;
+ auto chunks = Builder.GetArrowBlockChunks(val1, isScalar, length);
+ UNIT_ASSERT_VALUES_EQUAL(chunks, 2);
+ UNIT_ASSERT(!isScalar);
+ UNIT_ASSERT_VALUES_EQUAL(length, 5);
+
+ ArrowArray arrs[2];
+ Builder.ExportArrowBlock(val1, 0, &arrs[0]);
+ Builder.ExportArrowBlock(val1, 1, &arrs[1]);
+ NUdf::TUnboxedValue val2 = Builder.ImportArrowBlock(arrs, 2, isScalar, *atype);
+ const auto& d2 = TArrowBlock::From(val2).GetDatum();
+ UNIT_ASSERT(d2.is_arraylike() && !d2.is_array());
+ UNIT_ASSERT_VALUES_EQUAL(d2.length(), 5);
+ UNIT_ASSERT_VALUES_EQUAL(d2.chunks().size(), 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(d2.chunks()[0]->data()->length, 3);
+ UNIT_ASSERT_VALUES_EQUAL(d2.chunks()[0]->data()->GetNullCount(), 0);
+ auto flat = d2.chunks()[0]->data()->GetValues<ui64>(1);
+ UNIT_ASSERT_VALUES_EQUAL(flat[0], 10);
+ UNIT_ASSERT_VALUES_EQUAL(flat[1], 20);
+ UNIT_ASSERT_VALUES_EQUAL(flat[2], 30);
+
+ UNIT_ASSERT_VALUES_EQUAL(d2.chunks()[1]->data()->length, 2);
+ UNIT_ASSERT_VALUES_EQUAL(d2.chunks()[1]->data()->GetNullCount(), 0);
+ flat = d2.chunks()[1]->data()->GetValues<ui64>(1);
+ UNIT_ASSERT_VALUES_EQUAL(flat[0], 40);
+ UNIT_ASSERT_VALUES_EQUAL(flat[1], 50);
+ }
+ }
+};
+
+UNIT_TEST_SUITE_REGISTRATION(TMiniKQLValueBuilderTest);
+
+}
+}
diff --git a/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..558b995f359
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,52 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-computation-no_llvm)
+target_compile_options(minikql-computation-no_llvm PRIVATE
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-computation-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-computation-no_llvm PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ cpp-actors-util
+ library-cpp-enumbitset
+ library-cpp-packedtypes
+ library-cpp-random_provider
+ library-cpp-time_provider
+ library-yql-minikql
+ yql-minikql-arrow
+ parser-pg_wrapper-interface
+ yql-public-udf
+ library-yql-utils
+ cpp-threading-future
+)
+target_sources(minikql-computation-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_builder.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_reader.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_transport.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_custom_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort.cpp
+)
diff --git a/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..c2fbea09031
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,53 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-computation-no_llvm)
+target_compile_options(minikql-computation-no_llvm PRIVATE
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-computation-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-computation-no_llvm PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ cpp-actors-util
+ library-cpp-enumbitset
+ library-cpp-packedtypes
+ library-cpp-random_provider
+ library-cpp-time_provider
+ library-yql-minikql
+ yql-minikql-arrow
+ parser-pg_wrapper-interface
+ yql-public-udf
+ library-yql-utils
+ cpp-threading-future
+)
+target_sources(minikql-computation-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_builder.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_reader.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_transport.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_custom_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort.cpp
+)
diff --git a/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..c2fbea09031
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,53 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-computation-no_llvm)
+target_compile_options(minikql-computation-no_llvm PRIVATE
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-computation-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-computation-no_llvm PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ cpp-actors-util
+ library-cpp-enumbitset
+ library-cpp-packedtypes
+ library-cpp-random_provider
+ library-cpp-time_provider
+ library-yql-minikql
+ yql-minikql-arrow
+ parser-pg_wrapper-interface
+ yql-public-udf
+ library-yql-utils
+ cpp-threading-future
+)
+target_sources(minikql-computation-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_builder.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_reader.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_transport.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_custom_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort.cpp
+)
diff --git a/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.txt b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..558b995f359
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/no_llvm/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,52 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-computation-no_llvm)
+target_compile_options(minikql-computation-no_llvm PRIVATE
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-computation-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-computation-no_llvm PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ cpp-actors-util
+ library-cpp-enumbitset
+ library-cpp-packedtypes
+ library-cpp-random_provider
+ library-cpp-time_provider
+ library-yql-minikql
+ yql-minikql-arrow
+ parser-pg_wrapper-interface
+ yql-public-udf
+ library-yql-utils
+ cpp-threading-future
+)
+target_sources(minikql-computation-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_builder.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_reader.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_block_transport.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_impl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_custom_list.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort.cpp
+)
diff --git a/ydb/library/yql/minikql/computation/no_llvm/ya.make b/ydb/library/yql/minikql/computation/no_llvm/ya.make
new file mode 100644
index 00000000000..7e86314b947
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/no_llvm/ya.make
@@ -0,0 +1,9 @@
+LIBRARY()
+
+CXXFLAGS(-DMKQL_DISABLE_CODEGEN)
+
+ADDINCL(GLOBAL ydb/library/yql/minikql/codegen/llvm_stub)
+
+INCLUDE(../ya.make.inc)
+
+END()
diff --git a/ydb/library/yql/minikql/computation/presort_ut.cpp b/ydb/library/yql/minikql/computation/presort_ut.cpp
new file mode 100644
index 00000000000..0a6e537e4e8
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/presort_ut.cpp
@@ -0,0 +1,649 @@
+#include "presort.h"
+
+#include <ydb/library/yql/minikql/mkql_alloc.h>
+#include <ydb/library/yql/minikql/mkql_string_util.h>
+#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
+#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/string/hex.h>
+
+using namespace std::literals::string_view_literals;
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
+#define TYPE_MAP(XX) \
+ XX(bool, Bool) \
+ XX(ui8, Uint8) \
+ XX(ui16, Uint16) \
+ XX(ui32, Uint32) \
+ XX(ui64, Uint64) \
+ XX(i8, Int8) \
+ XX(i16, Int16) \
+ XX(i32, Int32) \
+ XX(i64, Int64) \
+ XX(float, Float) \
+ XX(double, Double)
+
+#define ALL_VALUES(xType, xName) \
+ xType xName;
+
+struct TSimpleTypes {
+ TYPE_MAP(ALL_VALUES)
+};
+#undef ALL_VALUES
+
+#define ADD_TYPE(xType, xName) \
+ codec.AddType(NUdf::EDataSlot::xName, isOptional, isDesc);
+
+void AddTypes(TPresortCodec& codec, bool isOptional, bool isDesc) {
+ TYPE_MAP(ADD_TYPE)
+}
+#undef ADD_TYPE
+
+#define ENCODE(xType, xName) \
+ encoder.Encode(NUdf::TUnboxedValuePod(values.xName));
+
+TStringBuf Encode(NKikimr::NMiniKQL::TPresortEncoder& encoder, const TSimpleTypes& values) {
+ encoder.Start();
+ TYPE_MAP(ENCODE)
+ return encoder.Finish();
+}
+#undef ENCODE
+
+#define DECODE(xType, xName) \
+ UNIT_ASSERT_EQUAL(decoder.Decode().Get<xType>(), values.xName);
+
+void Decode(TPresortDecoder& decoder, TStringBuf input, const TSimpleTypes& values) {
+ decoder.Start(input);
+ TYPE_MAP(DECODE)
+ decoder.Finish();
+}
+#undef DECODE
+
+#undef TYPE_MAP
+
+struct TPresortTest {
+ TScopedAlloc Alloc;
+ TMemoryUsageInfo MemInfo;
+
+ TPresortTest()
+ : Alloc(__LOCATION__)
+ , MemInfo("Memory")
+ {}
+
+ template <typename T>
+ void ValidateEncoding(bool isDesc, T value, const TString& hex) {
+ TPresortEncoder encoder;
+ encoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc);
+
+ TPresortDecoder decoder;
+ decoder.AddType(NUdf::TDataType<T>::Slot, false, isDesc);
+
+ encoder.Start();
+ encoder.Encode(NUdf::TUnboxedValuePod(value));
+ auto bytes = encoder.Finish();
+
+ UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
+
+ decoder.Start(bytes);
+ auto decoded = decoder.Decode().Get<T>();
+ decoder.Finish();
+
+ UNIT_ASSERT_EQUAL(decoded, value);
+ };
+
+ template <NUdf::EDataSlot Slot>
+ void ValidateEncoding(bool isDesc, TStringBuf value, const TString& hex) {
+ TPresortEncoder encoder;
+ encoder.AddType(Slot, false, isDesc);
+
+ TPresortDecoder decoder;
+ decoder.AddType(Slot, false, isDesc);
+
+ encoder.Start();
+ encoder.Encode(NUdf::TUnboxedValue(MakeString(NUdf::TStringRef(value))));
+ auto bytes = encoder.Finish();
+
+ UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
+
+ decoder.Start(bytes);
+ auto uv = decoder.Decode();
+ decoder.Finish();
+
+ auto stringRef = uv.AsStringRef();
+ auto decoded = TStringBuf(stringRef.Data(), stringRef.Size());
+
+ UNIT_ASSERT_EQUAL(decoded, value);
+ }
+
+ template <NUdf::EDataSlot Slot, typename T>
+ void ValidateEncoding(bool isDesc, const std::pair<T, ui16>& value, const TString& hex) {
+ TPresortEncoder encoder;
+ encoder.AddType(Slot, false, isDesc);
+
+ TPresortDecoder decoder;
+ decoder.AddType(Slot, false, isDesc);
+
+ NUdf::TUnboxedValuePod uv(value.first);
+ uv.SetTimezoneId(value.second);
+
+ encoder.Start();
+ encoder.Encode(uv);
+ auto bytes = encoder.Finish();
+
+ UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
+
+ decoder.Start(bytes);
+ auto decoded = decoder.Decode();
+ decoder.Finish();
+
+ UNIT_ASSERT_EQUAL(decoded.Get<T>(), value.first);
+ UNIT_ASSERT_EQUAL(decoded.GetTimezoneId(), value.second);
+ };
+
+ void ValidateEncoding(bool isDesc, NYql::NDecimal::TInt128 value, const TString& hex) {
+ TPresortEncoder encoder;
+ encoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc);
+
+ TPresortDecoder decoder;
+ decoder.AddType(NUdf::EDataSlot::Decimal, false, isDesc);
+
+ encoder.Start();
+ encoder.Encode(NUdf::TUnboxedValuePod(value));
+ auto bytes = encoder.Finish();
+
+ UNIT_ASSERT_EQUAL(HexEncode(bytes.data(), bytes.size()), hex);
+
+ decoder.Start(bytes);
+ auto decoded = decoder.Decode().GetInt128();
+ decoder.Finish();
+
+ UNIT_ASSERT_EQUAL(decoded, value);
+ };
+
+ template <typename T>
+ void ValidateEncoding(const TVector<T>& values) {
+ for (auto& value : values) {
+ ValidateEncoding(false, std::get<0>(value), std::get<1>(value));
+ ValidateEncoding(true, std::get<0>(value), std::get<2>(value));
+ }
+ }
+
+ template <NUdf::EDataSlot Slot, typename T>
+ void ValidateEncoding(const TVector<T>& values) {
+ for (auto& value : values) {
+ ValidateEncoding<Slot>(false, std::get<0>(value), std::get<1>(value));
+ ValidateEncoding<Slot>(true, std::get<0>(value), std::get<2>(value));
+ }
+ }
+};
+
+}
+
+Y_UNIT_TEST_SUITE(TPresortCodecTest) {
+
+Y_UNIT_TEST(SimpleTypes) {
+ TPresortTest test;
+
+ TSimpleTypes values = {false, 1u, 2u, 3u, 4u, 5, 6, 7, 8, 9.f, 10.0};
+
+ auto validateSimpleTypes = [&] (bool isOptional, bool isDesc) {
+ TPresortEncoder encoder;
+ AddTypes(encoder, isOptional, isDesc);
+
+ TPresortDecoder decoder;
+ AddTypes(decoder, isOptional, isDesc);
+
+ auto bytes = Encode(encoder, values);
+ Decode(decoder, bytes, values);
+ };
+
+ validateSimpleTypes(false, false);
+ validateSimpleTypes(false, true);
+ validateSimpleTypes(true, false);
+ validateSimpleTypes(true, true);
+}
+
+
+Y_UNIT_TEST(Bool) {
+ const TVector<std::tuple<bool, TString, TString>> values = {
+ {false, "00", "FF"},
+ {true, "01", "FE"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(Int8) {
+ const TVector<std::tuple<i8, TString, TString>> values = {
+ {-0x80, "00", "FF"},
+ {-1, "7F", "80"},
+ {0, "80", "7F"},
+ {1, "81", "7E"},
+ {0x7F, "FF", "00"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(Uint8) {
+ const TVector<std::tuple<ui8, TString, TString>> values = {
+ {0u, "00", "FF"},
+ {0x80u, "80", "7F"},
+ {0xFFu, "FF", "00"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(Int16) {
+ const TVector<std::tuple<i16, TString, TString>> values = {
+ {-0x8000, "0000", "FFFF"},
+ {-1, "7FFF", "8000"},
+ {0, "8000", "7FFF"},
+ {1, "8001", "7FFE"},
+ {0x7FFF, "FFFF", "0000"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(Uint16) {
+ const TVector<std::tuple<ui16, TString, TString>> values = {
+ {0, "0000", "FFFF"},
+ {0x8000u, "8000", "7FFF"},
+ {0xFFFFu, "FFFF", "0000"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(Int32) {
+ const TVector<std::tuple<i32, TString, TString>> values = {
+ {-0x80000000, "00000000", "FFFFFFFF"},
+ {-1, "7FFFFFFF", "80000000"},
+ {0, "80000000", "7FFFFFFF"},
+ {1, "80000001", "7FFFFFFE"},
+ {0x7FFFFFFF, "FFFFFFFF", "00000000"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(Uint32) {
+ const TVector<std::tuple<ui32, TString, TString>> values = {
+ {0u, "00000000", "FFFFFFFF"},
+ {0x80000000u, "80000000", "7FFFFFFF"},
+ {0xFFFFFFFFu, "FFFFFFFF", "00000000"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(Int64) {
+ const TVector<std::tuple<i64, TString, TString>> values = {
+ {-0x8000000000000000, "0000000000000000", "FFFFFFFFFFFFFFFF"},
+ {-1, "7FFFFFFFFFFFFFFF", "8000000000000000"},
+ {0, "8000000000000000", "7FFFFFFFFFFFFFFF"},
+ {1, "8000000000000001", "7FFFFFFFFFFFFFFE"},
+ {0x7FFFFFFFFFFFFFFF, "FFFFFFFFFFFFFFFF", "0000000000000000"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(Uint64) {
+ const TVector<std::tuple<ui64, TString, TString>> values = {
+ {0u, "0000000000000000", "FFFFFFFFFFFFFFFF"},
+ {0x8000000000000000u, "8000000000000000", "7FFFFFFFFFFFFFFF"},
+ {0xFFFFFFFFFFFFFFFFu, "FFFFFFFFFFFFFFFF", "0000000000000000"}
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(Float) {
+ using TLimits = std::numeric_limits<float>;
+
+ const TVector<std::tuple<float, TString, TString>> values = {
+ {-TLimits::infinity(), "00", "FF"},
+ {-TLimits::max(), "0100800000", "FEFF7FFFFF"},
+ {-1.f, "01407FFFFF", "FEBF800000"},
+ {-TLimits::min(), "017F7FFFFF", "FE80800000"},
+ {-TLimits::min()/8.f, "017FEFFFFF", "FE80100000"},
+ {0.f, "02", "FD"},
+ {TLimits::min()/8.f, "0300100000", "FCFFEFFFFF"},
+ {TLimits::min(), "0300800000", "FCFF7FFFFF"},
+ {1.f, "033F800000", "FCC07FFFFF"},
+ {TLimits::max(), "037F7FFFFF", "FC80800000"},
+ {TLimits::infinity(), "04", "FB"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(Double) {
+ using TLimits = std::numeric_limits<double>;
+
+ const TVector<std::tuple<double, TString, TString>> values = {
+ {-TLimits::infinity(), "00", "FF"},
+ {-TLimits::max(), "010010000000000000", "FEFFEFFFFFFFFFFFFF"},
+ {-1., "01400FFFFFFFFFFFFF", "FEBFF0000000000000"},
+ {-TLimits::min(), "017FEFFFFFFFFFFFFF", "FE8010000000000000"},
+ {-TLimits::min()/8., "017FFDFFFFFFFFFFFF", "FE8002000000000000"},
+ {0., "02", "FD"},
+ {TLimits::min()/8., "030002000000000000", "FCFFFDFFFFFFFFFFFF"},
+ {TLimits::min(), "030010000000000000", "FCFFEFFFFFFFFFFFFF"},
+ {1., "033FF0000000000000", "FCC00FFFFFFFFFFFFF"},
+ {TLimits::max(), "037FEFFFFFFFFFFFFF", "FC8010000000000000"},
+ {TLimits::infinity(), "04", "FB"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(String) {
+ const TVector<std::tuple<TStringBuf, TString, TString>> values = {
+ {TStringBuf(""), "00", "FF"},
+ {"\x00"sv, "1F00000000000000000000000000000001",
+ "E0FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ {"\x01", "1F01000000000000000000000000000001",
+ "E0FEFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ {"0", "1F30000000000000000000000000000001",
+ "E0CFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ {"0123", "1F30313233000000000000000000000004",
+ "E0CFCECDCCFFFFFFFFFFFFFFFFFFFFFFFB"},
+ {"0123456789abcde", "1F3031323334353637383961626364650F",
+ "E0CFCECDCCCBCAC9C8C7C69E9D9C9B9AF0"},
+ {"a", "1F61000000000000000000000000000001",
+ "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ {"a\x00"sv, "1F61000000000000000000000000000002",
+ "E09EFFFFFFFFFFFFFFFFFFFFFFFFFFFFFD"},
+ {"abc", "1F61626300000000000000000000000003",
+ "E09E9D9CFFFFFFFFFFFFFFFFFFFFFFFFFC"},
+ {"b", "1F62000000000000000000000000000001",
+ "E09DFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ };
+ TPresortTest().ValidateEncoding<NUdf::EDataSlot::String>(values);
+}
+
+Y_UNIT_TEST(Uuid) {
+ const TVector<std::tuple<TStringBuf, TString, TString>> values = {
+ {"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0"sv,
+ "00000000000000000000000000000000",
+ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF"},
+ {"\0\0\0\0\0\0\0\0\0\0\0\0\0\0\0\1"sv,
+ "00000000000000000000000000000001",
+ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFE"},
+ {"\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff\xff",
+ "FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF",
+ "00000000000000000000000000000000"},
+ };
+ TPresortTest().ValidateEncoding<NUdf::EDataSlot::Uuid>(values);
+}
+
+Y_UNIT_TEST(TzDate) {
+ const TVector<std::tuple<std::pair<ui16, ui16>, TString, TString>> values = {
+ {{0u, 0u}, "00000000", "FFFFFFFF"},
+ {{0u, 1u}, "00000001", "FFFFFFFE"},
+ {{1u, 0u}, "00010000", "FFFEFFFF"},
+ {{NUdf::MAX_DATE, 0u}, "C2090000", "3DF6FFFF"},
+ };
+ TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDate>(values);
+}
+
+Y_UNIT_TEST(TzDatetime) {
+ const TVector<std::tuple<std::pair<ui32, ui16>, TString, TString>> values = {
+ {{0u, 0u}, "000000000000", "FFFFFFFFFFFF"},
+ {{0u, 1u}, "000000000001", "FFFFFFFFFFFE"},
+ {{1u, 0u}, "000000010000", "FFFFFFFEFFFF"},
+ {{NUdf::MAX_DATETIME, 0u}, "FFCEDD800000", "0031227FFFFF"},
+ };
+ TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzDatetime>(values);
+}
+
+Y_UNIT_TEST(TzTimestamp) {
+ const TVector<std::tuple<std::pair<ui64, ui16>, TString, TString>> values = {
+ {{0u, 0u}, "00000000000000000000", "FFFFFFFFFFFFFFFFFFFF"},
+ {{0u, 1u}, "00000000000000000001", "FFFFFFFFFFFFFFFFFFFE"},
+ {{1u, 0u}, "00000000000000010000", "FFFFFFFFFFFFFFFEFFFF"},
+ {{NUdf::MAX_TIMESTAMP, 0u}, "000F3F52435260000000", "FFF0C0ADBCAD9FFFFFFF"},
+ };
+ TPresortTest().ValidateEncoding<NUdf::EDataSlot::TzTimestamp>(values);
+}
+
+Y_UNIT_TEST(Decimal) {
+ const TVector<std::tuple<NYql::NDecimal::TInt128, TString, TString>> values = {
+ {-NYql::NDecimal::Nan(),
+ "00",
+ "FF"},
+ {-NYql::NDecimal::Inf(),
+ "01",
+ "FE"},
+ {NYql::NDecimal::TInt128(-1),
+ "7F",
+ "8101"},
+ {NYql::NDecimal::TInt128(0),
+ "80",
+ "80"},
+ {NYql::NDecimal::TInt128(1),
+ "8101",
+ "7F"},
+ {NYql::NDecimal::Inf(),
+ "FE",
+ "01"},
+ {NYql::NDecimal::Nan(),
+ "FF",
+ "00"},
+ };
+ TPresortTest().ValidateEncoding(values);
+}
+
+Y_UNIT_TEST(GenericVoid) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ auto type = env.GetVoid()->GetType();
+ NUdf::TUnboxedValue value = NUdf::TUnboxedValuePod::Void();
+ TGenericPresortEncoder encoder(type);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf(""));
+}
+
+Y_UNIT_TEST(GenericBool) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ auto type = TDataType::Create(NUdf::TDataType<bool>::Id, env);
+ NUdf::TUnboxedValue value = NUdf::TUnboxedValuePod(true);
+ TGenericPresortEncoder encoder(type);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01"));
+ buf = encoder.Encode(value, true);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\xFE"));
+}
+
+Y_UNIT_TEST(GenericNumber) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ auto type = TDataType::Create(NUdf::TDataType<ui32>::Id, env);
+ NUdf::TUnboxedValue value = NUdf::TUnboxedValuePod(ui32(1234));
+ TGenericPresortEncoder encoder(type);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00\x00\x04\xD2"sv));
+}
+
+Y_UNIT_TEST(GenericString) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ auto type = TDataType::Create(NUdf::TDataType<char*>::Id, env);
+ NUdf::TUnboxedValue value = MakeString("ALongStringExample");
+ TGenericPresortEncoder encoder(type);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x1F" "ALongStringExam\x1Fple\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x00\x03"sv));
+}
+
+Y_UNIT_TEST(GenericOptional) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ auto type = TOptionalType::Create(TDataType::Create(NUdf::TDataType<bool>::Id, env), env);
+ NUdf::TUnboxedValue value = NUdf::TUnboxedValuePod(true);
+ TGenericPresortEncoder encoder(type);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01"));
+ value = {};
+ buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv));
+}
+
+Y_UNIT_TEST(NestedOptional) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ // Int32???
+ auto type =
+ TOptionalType::Create(TOptionalType::Create(TOptionalType::Create(TDataType::Create(NUdf::TDataType<i32>::Id, env), env), env), env);
+ TGenericPresortEncoder encoder(type);
+
+ NUdf::TUnboxedValue null = {};
+ auto buf = encoder.Encode(null, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv));
+
+ auto justNull = null.MakeOptional();
+ buf = encoder.Encode(justNull, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00"sv));
+
+ auto justJustNull = justNull.MakeOptional();
+ buf = encoder.Encode(justJustNull, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x00"sv));
+
+
+ auto zero = NUdf::TUnboxedValuePod(0).MakeOptional().MakeOptional().MakeOptional();
+ buf = encoder.Encode(zero, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x01\x80\x00\x00\x00"sv));
+}
+
+
+Y_UNIT_TEST(GenericList) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ auto type = TListType::Create(TDataType::Create(NUdf::TDataType<bool>::Id, env), env);
+ TMemoryUsageInfo memInfo("test");
+ THolderFactory holderFactory(alloc.Ref(), memInfo);
+ auto value = holderFactory.GetEmptyContainer();
+ TGenericPresortEncoder encoder(type);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv));
+ NUdf::TUnboxedValue* items;
+ value = holderFactory.CreateDirectArrayHolder(1, items);
+ items[0] = NUdf::TUnboxedValuePod(true);
+ buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x00"sv));
+ value = holderFactory.CreateDirectArrayHolder(2, items);
+ items[0] = NUdf::TUnboxedValuePod(true);
+ items[1] = NUdf::TUnboxedValuePod(false);
+ buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x01\x01\x00\x00"sv));
+}
+
+Y_UNIT_TEST(GenericTuple) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ TType* tupleTypes[2];
+ tupleTypes[0] = TDataType::Create(NUdf::TDataType<bool>::Id, env);
+ tupleTypes[1] = TDataType::Create(NUdf::TDataType<ui32>::Id, env);
+ auto type = TTupleType::Create(2, tupleTypes, env);
+ TMemoryUsageInfo memInfo("test");
+ THolderFactory holderFactory(alloc.Ref(), memInfo);
+ NUdf::TUnboxedValue* items;
+ auto value = holderFactory.CreateDirectArrayHolder(2, items);
+ items[0] = NUdf::TUnboxedValuePod(true);
+ items[1] = NUdf::TUnboxedValuePod(ui32(1234));
+ TGenericPresortEncoder encoder(type);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2"sv));
+}
+
+Y_UNIT_TEST(GenericStruct) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ TStructMember structTypes[2];
+ structTypes[0] = TStructMember("A", TDataType::Create(NUdf::TDataType<bool>::Id, env));
+ structTypes[1] = TStructMember("B", TDataType::Create(NUdf::TDataType<ui32>::Id, env));
+ auto type = TStructType::Create(2, structTypes, env);
+ TMemoryUsageInfo memInfo("test");
+ THolderFactory holderFactory(alloc.Ref(), memInfo);
+ NUdf::TUnboxedValue* items;
+ auto value = holderFactory.CreateDirectArrayHolder(2, items);
+ items[0] = NUdf::TUnboxedValuePod(true);
+ items[1] = NUdf::TUnboxedValuePod(ui32(1234));
+ TGenericPresortEncoder encoder(type);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2"sv));
+}
+
+Y_UNIT_TEST(GenericTupleVariant) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ TType* tupleTypes[2];
+ tupleTypes[0] = TDataType::Create(NUdf::TDataType<bool>::Id, env);
+ tupleTypes[1] = TDataType::Create(NUdf::TDataType<ui32>::Id, env);
+ auto underlying = TTupleType::Create(2, tupleTypes, env);
+ auto type = TVariantType::Create(underlying, env);
+ TMemoryUsageInfo memInfo("test");
+ THolderFactory holderFactory(alloc.Ref(), memInfo);
+ TGenericPresortEncoder encoder(type);
+ auto value = holderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(true), 0);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00\x01"sv));
+ value = holderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(ui32(1234)), 1);
+ buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2"sv));
+}
+
+Y_UNIT_TEST(GenericStructVariant) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ TStructMember structTypes[2];
+ structTypes[0] = TStructMember("A", TDataType::Create(NUdf::TDataType<bool>::Id, env));
+ structTypes[1] = TStructMember("B", TDataType::Create(NUdf::TDataType<ui32>::Id, env));
+ auto underlying = TStructType::Create(2, structTypes, env);
+ auto type = TVariantType::Create(underlying, env);
+ TMemoryUsageInfo memInfo("test");
+ THolderFactory holderFactory(alloc.Ref(), memInfo);
+ TGenericPresortEncoder encoder(type);
+ auto value = holderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(true), 0);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00\x01"sv));
+ value = holderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(ui32(1234)), 1);
+ buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2"sv));
+}
+
+Y_UNIT_TEST(GenericDict) {
+ TScopedAlloc alloc(__LOCATION__);
+ TTypeEnvironment env(alloc);
+ auto type = TDictType::Create(TDataType::Create(NUdf::TDataType<ui32>::Id, env),
+ TDataType::Create(NUdf::TDataType<bool>::Id, env), env);
+ TKeyTypes keyTypes;
+ bool isTuple;
+ bool encoded;
+ bool useIHash;
+ GetDictionaryKeyTypes(type->GetKeyType(), keyTypes, isTuple, encoded, useIHash);
+ UNIT_ASSERT(!isTuple);
+ UNIT_ASSERT(!encoded);
+ UNIT_ASSERT(!useIHash);
+ TMemoryUsageInfo memInfo("test");
+ THolderFactory holderFactory(alloc.Ref(), memInfo);
+ auto value = holderFactory.GetEmptyContainer();
+ TGenericPresortEncoder encoder(type);
+ auto buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x00"sv));
+ value = holderFactory.CreateDirectHashedDictHolder([](TValuesDictHashMap& map) {
+ map.emplace(NUdf::TUnboxedValuePod(ui32(1234)), NUdf::TUnboxedValuePod(true));
+ }, keyTypes, false, true, nullptr, nullptr, nullptr);
+ buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2\x01\x00"sv));
+ value = holderFactory.CreateDirectHashedDictHolder([](TValuesDictHashMap& map) {
+ map.emplace(NUdf::TUnboxedValuePod(ui32(5678)), NUdf::TUnboxedValuePod(false));
+ map.emplace(NUdf::TUnboxedValuePod(ui32(1234)), NUdf::TUnboxedValuePod(true));
+ }, keyTypes, false, true, nullptr, nullptr, nullptr);
+ buf = encoder.Encode(value, false);
+ UNIT_ASSERT_NO_DIFF(buf, TStringBuf("\x01\x00\x00\x04\xD2\x01\x01\x00\x00\x16\x2E\x00\x00"sv));
+}
+
+}
+
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/computation/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/computation/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..01dba3f99d8
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,90 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-computation-ut)
+target_compile_options(ydb-library-yql-minikql-computation-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-computation-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-computation-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ minikql-computation-llvm
+ libs-apache-arrow
+ cpp-threading-local_executor
+ minikql-comp_nodes-llvm
+ yql-parser-pg_wrapper
+ udf-service-exception_policy
+ yql-dq-proto
+)
+target_link_options(ydb-library-yql-minikql-computation-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-minikql-computation-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-computation-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-computation-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-computation-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-computation-ut)
diff --git a/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..67eac9242ad
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,93 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-computation-ut)
+target_compile_options(ydb-library-yql-minikql-computation-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-computation-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-computation-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ minikql-computation-llvm
+ libs-apache-arrow
+ cpp-threading-local_executor
+ minikql-comp_nodes-llvm
+ yql-parser-pg_wrapper
+ udf-service-exception_policy
+ yql-dq-proto
+)
+target_link_options(ydb-library-yql-minikql-computation-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-computation-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-computation-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-computation-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-computation-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-minikql-computation-ut)
diff --git a/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..436ad6ceac5
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,95 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-computation-ut)
+target_compile_options(ydb-library-yql-minikql-computation-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-computation-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-computation-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ minikql-computation-llvm
+ libs-apache-arrow
+ cpp-threading-local_executor
+ minikql-comp_nodes-llvm
+ yql-parser-pg_wrapper
+ udf-service-exception_policy
+ yql-dq-proto
+)
+target_link_options(ydb-library-yql-minikql-computation-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-computation-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-computation-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-computation-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-computation-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-minikql-computation-ut)
diff --git a/ydb/library/yql/minikql/computation/ut/CMakeLists.txt b/ydb/library/yql/minikql/computation/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/computation/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/computation/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..4244aa874b3
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,83 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-computation-ut)
+target_compile_options(ydb-library-yql-minikql-computation-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-computation-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-computation-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ minikql-computation-llvm
+ libs-apache-arrow
+ cpp-threading-local_executor
+ minikql-comp_nodes-llvm
+ yql-parser-pg_wrapper
+ udf-service-exception_policy
+ yql-dq-proto
+)
+target_sources(ydb-library-yql-minikql-computation-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_list_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_node_graph_saveload_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_computation_pattern_cache_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/mkql_value_builder_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/computation/presort_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-computation-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-computation-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-computation-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-computation-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-computation-ut)
diff --git a/ydb/library/yql/minikql/computation/ut/ya.make b/ydb/library/yql/minikql/computation/ut/ya.make
new file mode 100644
index 00000000000..876dc95ff82
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/ut/ya.make
@@ -0,0 +1,38 @@
+UNITTEST_FOR(ydb/library/yql/minikql/computation/llvm)
+
+FORK_SUBTESTS()
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ TIMEOUT(1800)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+SRCDIR(ydb/library/yql/minikql/computation)
+
+SRCS(
+ mkql_computation_node_pack_ut.cpp
+ mkql_computation_node_list_ut.cpp
+ mkql_computation_node_dict_ut.cpp
+ mkql_computation_node_graph_saveload_ut.cpp
+ mkql_computation_pattern_cache_ut.cpp
+ mkql_validate_ut.cpp
+ mkql_value_builder_ut.cpp
+ presort_ut.cpp
+)
+
+PEERDIR(
+ contrib/libs/apache/arrow
+ library/cpp/threading/local_executor
+ ydb/library/yql/minikql/comp_nodes/llvm
+ ydb/library/yql/parser/pg_wrapper
+ ydb/library/yql/public/udf/service/exception_policy
+ ydb/library/yql/dq/proto
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/minikql/computation/ya.make b/ydb/library/yql/minikql/computation/ya.make
new file mode 100644
index 00000000000..424ecfb86d0
--- /dev/null
+++ b/ydb/library/yql/minikql/computation/ya.make
@@ -0,0 +1,8 @@
+RECURSE(
+ llvm
+ no_llvm
+)
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/minikql/dom/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/dom/CMakeLists.darwin-x86_64.txt
index 4d9af8a9e92..5ab46c00cdf 100644
--- a/ydb/library/yql/minikql/dom/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/minikql/dom/CMakeLists.darwin-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-dom)
target_compile_options(yql-minikql-dom PRIVATE
diff --git a/ydb/library/yql/minikql/dom/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/dom/CMakeLists.linux-aarch64.txt
index 5d15504310f..c6d55bf198d 100644
--- a/ydb/library/yql/minikql/dom/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/minikql/dom/CMakeLists.linux-aarch64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-dom)
target_compile_options(yql-minikql-dom PRIVATE
diff --git a/ydb/library/yql/minikql/dom/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/dom/CMakeLists.linux-x86_64.txt
index 5d15504310f..c6d55bf198d 100644
--- a/ydb/library/yql/minikql/dom/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/minikql/dom/CMakeLists.linux-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(yql-minikql-dom)
target_compile_options(yql-minikql-dom PRIVATE
diff --git a/ydb/library/yql/minikql/dom/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/dom/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..2fd86a151a5
--- /dev/null
+++ b/ydb/library/yql/minikql/dom/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,74 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-dom-ut)
+target_compile_options(ydb-library-yql-minikql-dom-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-dom-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom
+)
+target_link_libraries(ydb-library-yql-minikql-dom-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-dom
+ minikql-computation-llvm
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-dom-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-minikql-dom-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/yson_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/json_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-dom-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-dom-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-dom-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-dom-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-dom-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-minikql-dom-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-dom-ut)
diff --git a/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..3fefeffccc1
--- /dev/null
+++ b/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,77 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-dom-ut)
+target_compile_options(ydb-library-yql-minikql-dom-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-dom-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom
+)
+target_link_libraries(ydb-library-yql-minikql-dom-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ yql-minikql-dom
+ minikql-computation-llvm
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-dom-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-dom-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/yson_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/json_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-dom-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-dom-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-dom-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-dom-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-dom-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-minikql-dom-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-minikql-dom-ut)
diff --git a/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..739b1984df5
--- /dev/null
+++ b/ydb/library/yql/minikql/dom/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,79 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-dom-ut)
+target_compile_options(ydb-library-yql-minikql-dom-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-dom-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom
+)
+target_link_libraries(ydb-library-yql-minikql-dom-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-dom
+ minikql-computation-llvm
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-dom-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-dom-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/yson_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/dom/ut/json_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-dom-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-dom-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-dom-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-dom-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-dom-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-minikql-dom-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-minikql-dom-ut)
diff --git a/ydb/library/yql/minikql/dom/ut/CMakeLists.txt b/ydb/library/yql/minikql/dom/ut/CMakeLists.txt
new file mode 100644
index 00000000000..606ff46b4be
--- /dev/null
+++ b/ydb/library/yql/minikql/dom/ut/CMakeLists.txt
@@ -0,0 +1,15 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/dom/ut/ya.make b/ydb/library/yql/minikql/dom/ut/ya.make
index 2386e8e0696..b6e80f6c7b0 100644
--- a/ydb/library/yql/minikql/dom/ut/ya.make
+++ b/ydb/library/yql/minikql/dom/ut/ya.make
@@ -1,5 +1,5 @@
IF (NOT WINDOWS)
- UNITTEST_FOR(yql/library/dom)
+ UNITTEST_FOR(ydb/library/yql/minikql/dom)
SRCS(
yson_ut.cpp
@@ -14,6 +14,7 @@ IF (NOT WINDOWS)
PEERDIR(
ydb/library/yql/minikql/computation/llvm
ydb/library/yql/public/udf/service/exception_policy
+ ydb/library/yql/sql/pg_dummy
)
YQL_LAST_ABI_VERSION()
diff --git a/ydb/library/yql/minikql/invoke_builtins/CMakeLists.txt b/ydb/library/yql/minikql/invoke_builtins/CMakeLists.txt
index e519707fa41..4716de1ef37 100644
--- a/ydb/library/yql/minikql/invoke_builtins/CMakeLists.txt
+++ b/ydb/library/yql/minikql/invoke_builtins/CMakeLists.txt
@@ -7,3 +7,5 @@
add_subdirectory(llvm)
+add_subdirectory(no_llvm)
+add_subdirectory(ut)
diff --git a/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp
new file mode 100644
index 00000000000..9420223b74e
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp
@@ -0,0 +1,51 @@
+#include "mkql_builtins.h"
+
+#include <ydb/library/yql/public/udf/udf_value.h>
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <array>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+static TFunctionParamMetadata AddUi32Metadata[] = {
+ { NUdf::TDataType<ui32>::Id, 0 }, // result
+ { NUdf::TDataType<ui32>::Id, 0 }, // first arg
+ { NUdf::TDataType<ui32>::Id, 0 }, // second arg
+ { 0, 0 }
+};
+
+static NUdf::TUnboxedValuePod AddUi32(const NUdf::TUnboxedValuePod* args)
+{
+ const ui32 first = args[0].Get<ui32>();
+ const ui32 second = args[1].Get<ui32>();
+ return NUdf::TUnboxedValuePod(first + second);
+}
+
+Y_UNIT_TEST_SUITE(TFunctionRegistryTest) {
+ Y_UNIT_TEST(TestRegistration) {
+ const auto functionRegistry = CreateBuiltinRegistry();
+ functionRegistry->Register("MyAdd", TFunctionDescriptor(AddUi32Metadata, &AddUi32));
+
+ const std::array<TArgType, 3U> argTypes ={{{ NUdf::TDataType<ui32>::Id, false }, { NUdf::TDataType<ui32>::Id, false }, { NUdf::TDataType<ui32>::Id, false }}};
+ auto op = functionRegistry->GetBuiltin("MyAdd", argTypes.data(), argTypes.size());
+ UNIT_ASSERT_EQUAL(op.Function, &AddUi32);
+ UNIT_ASSERT_EQUAL(op.ResultAndArgs[0].SchemeType, NUdf::TDataType<ui32>::Id);
+ UNIT_ASSERT_EQUAL(op.ResultAndArgs[0].Flags, 0);
+ UNIT_ASSERT_EQUAL(op.ResultAndArgs[1].SchemeType, NUdf::TDataType<ui32>::Id);
+ UNIT_ASSERT_EQUAL(op.ResultAndArgs[1].Flags, 0);
+ UNIT_ASSERT_EQUAL(op.ResultAndArgs[2].SchemeType, NUdf::TDataType<ui32>::Id);
+ UNIT_ASSERT_EQUAL(op.ResultAndArgs[2].Flags, 0);
+ UNIT_ASSERT_EQUAL(op.ResultAndArgs[3].SchemeType, 0);
+
+ const NUdf::TUnboxedValuePod args[2] = {NUdf::TUnboxedValuePod(ui32(2)), NUdf::TUnboxedValuePod(ui32(3))};
+
+ auto result = op.Function(&args[0]);
+ UNIT_ASSERT_EQUAL(result.Get<ui32>(), 5);
+ }
+}
+
+} // namespace NMiniKQL
+
+} // namespace NKikimr
diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..cd35db266b6
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,69 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-invoke_builtins-no_llvm)
+target_compile_options(minikql-invoke_builtins-no_llvm PRIVATE
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-invoke_builtins-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-invoke_builtins-no_llvm PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-pop_count
+ ydb-library-binary_json
+ library-yql-minikql
+ yql-minikql-arrow
+ yql-public-udf
+ libs-apache-arrow
+ minikql-computation-no_llvm
+)
+target_sources(minikql-invoke_builtins-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_add.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitand.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitnot.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitxor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_codegen.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_concat.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_countbits.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_decimal.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_dec.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_div.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_find.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_inc.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_invprestr.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_max.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_min.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_minus.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mod.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mul.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_nanvl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotleft.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotright.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftleft.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftright.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_string_kernels.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_sub.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_substring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_with.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_equals.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_not_equals.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less_or_equal.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater_or_equal.cpp
+)
diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..c71c6c41366
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,70 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-invoke_builtins-no_llvm)
+target_compile_options(minikql-invoke_builtins-no_llvm PRIVATE
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-invoke_builtins-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-invoke_builtins-no_llvm PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-pop_count
+ ydb-library-binary_json
+ library-yql-minikql
+ yql-minikql-arrow
+ yql-public-udf
+ libs-apache-arrow
+ minikql-computation-no_llvm
+)
+target_sources(minikql-invoke_builtins-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_add.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitand.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitnot.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitxor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_codegen.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_concat.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_countbits.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_decimal.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_dec.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_div.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_find.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_inc.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_invprestr.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_max.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_min.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_minus.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mod.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mul.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_nanvl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotleft.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotright.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftleft.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftright.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_string_kernels.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_sub.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_substring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_with.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_equals.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_not_equals.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less_or_equal.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater_or_equal.cpp
+)
diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..c71c6c41366
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,70 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-invoke_builtins-no_llvm)
+target_compile_options(minikql-invoke_builtins-no_llvm PRIVATE
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-invoke_builtins-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-invoke_builtins-no_llvm PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-pop_count
+ ydb-library-binary_json
+ library-yql-minikql
+ yql-minikql-arrow
+ yql-public-udf
+ libs-apache-arrow
+ minikql-computation-no_llvm
+)
+target_sources(minikql-invoke_builtins-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_add.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitand.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitnot.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitxor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_codegen.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_concat.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_countbits.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_decimal.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_dec.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_div.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_find.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_inc.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_invprestr.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_max.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_min.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_minus.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mod.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mul.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_nanvl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotleft.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotright.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftleft.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftright.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_string_kernels.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_sub.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_substring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_with.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_equals.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_not_equals.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less_or_equal.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater_or_equal.cpp
+)
diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.txt b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..cd35db266b6
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,69 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_library(minikql-invoke_builtins-no_llvm)
+target_compile_options(minikql-invoke_builtins-no_llvm PRIVATE
+ -DMKQL_DISABLE_CODEGEN
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(minikql-invoke_builtins-no_llvm PUBLIC
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/codegen/llvm_stub
+)
+target_link_libraries(minikql-invoke_builtins-no_llvm PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-pop_count
+ ydb-library-binary_json
+ library-yql-minikql
+ yql-minikql-arrow
+ yql-public-udf
+ libs-apache-arrow
+ minikql-computation-no_llvm
+)
+target_sources(minikql-invoke_builtins-no_llvm PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_abs.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_add.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitand.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitnot.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_bitxor.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_byteat.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_codegen.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_concat.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_convert.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_countbits.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_decimal.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_dec.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_div.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_find.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_inc.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_invprestr.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_max.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_min.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_minus.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mod.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_mul.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_nanvl.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_plus.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotleft.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_rotright.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftleft.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_shiftright.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_string_kernels.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_sub.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_substring.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_with.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_equals.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_not_equals.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_less_or_equal.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_greater_or_equal.cpp
+)
diff --git a/ydb/library/yql/minikql/invoke_builtins/no_llvm/ya.make b/ydb/library/yql/minikql/invoke_builtins/no_llvm/ya.make
new file mode 100644
index 00000000000..0de0a0d6fd7
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/no_llvm/ya.make
@@ -0,0 +1,12 @@
+LIBRARY()
+
+CXXFLAGS(-DMKQL_DISABLE_CODEGEN)
+
+ADDINCL(GLOBAL ydb/library/yql/minikql/codegen/llvm_stub)
+
+INCLUDE(../ya.make.inc)
+
+PEERDIR(ydb/library/yql/minikql/computation/no_llvm)
+
+END()
+
diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..29bde95f2eb
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,80 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-invoke_builtins-ut)
+target_compile_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-invoke_builtins-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ minikql-invoke_builtins-llvm
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-invoke_builtins-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-invoke_builtins-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-invoke_builtins-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-invoke_builtins-ut)
diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..6be7f1567aa
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,83 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-invoke_builtins-ut)
+target_compile_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-invoke_builtins-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ minikql-invoke_builtins-llvm
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-invoke_builtins-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-invoke_builtins-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-invoke_builtins-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-minikql-invoke_builtins-ut)
diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..74923c7ad7d
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,85 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-invoke_builtins-ut)
+target_compile_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-invoke_builtins-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ minikql-invoke_builtins-llvm
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-invoke_builtins-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-invoke_builtins-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-invoke_builtins-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-minikql-invoke_builtins-ut)
diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.txt b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..a3ad9ceabdc
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,73 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-invoke_builtins-ut)
+target_compile_options(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/llvm
+)
+target_link_libraries(ydb-library-yql-minikql-invoke_builtins-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ minikql-invoke_builtins-llvm
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+)
+target_sources(ydb-library-yql-minikql-invoke_builtins-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/invoke_builtins/mkql_builtins_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 10
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-invoke_builtins-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-invoke_builtins-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-invoke_builtins-ut
+ PROPERTY
+ TIMEOUT
+ 600
+)
+target_allocator(ydb-library-yql-minikql-invoke_builtins-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-invoke_builtins-ut)
diff --git a/ydb/library/yql/minikql/invoke_builtins/ut/ya.make b/ydb/library/yql/minikql/invoke_builtins/ut/ya.make
new file mode 100644
index 00000000000..eb88ec513d5
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/ut/ya.make
@@ -0,0 +1,28 @@
+UNITTEST_FOR(ydb/library/yql/minikql/invoke_builtins/llvm)
+
+FORK_SUBTESTS()
+
+IF (SANITIZER_TYPE OR WITH_VALGRIND)
+ TIMEOUT(1800)
+ SIZE(LARGE)
+ TAG(ya:fat)
+ELSE()
+ TIMEOUT(600)
+ SIZE(MEDIUM)
+ENDIF()
+
+PEERDIR(
+ ydb/library/yql/public/udf
+ ydb/library/yql/public/udf/service/exception_policy
+ ydb/library/yql/sql/pg_dummy
+)
+
+YQL_LAST_ABI_VERSION()
+
+SRCDIR(ydb/library/yql/minikql/invoke_builtins)
+
+SRCS(
+ mkql_builtins_ut.cpp
+)
+
+END()
diff --git a/ydb/library/yql/minikql/invoke_builtins/ya.make b/ydb/library/yql/minikql/invoke_builtins/ya.make
new file mode 100644
index 00000000000..424ecfb86d0
--- /dev/null
+++ b/ydb/library/yql/minikql/invoke_builtins/ya.make
@@ -0,0 +1,8 @@
+RECURSE(
+ llvm
+ no_llvm
+)
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/minikql/jsonpath/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/jsonpath/CMakeLists.darwin-x86_64.txt
index bb96e932dc2..5224e6130f6 100644
--- a/ydb/library/yql/minikql/jsonpath/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/minikql/jsonpath/CMakeLists.darwin-x86_64.txt
@@ -7,6 +7,7 @@
add_subdirectory(benchmark)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-aarch64.txt
index 09049a15249..a07d8bcad0a 100644
--- a/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-aarch64.txt
@@ -7,6 +7,7 @@
add_subdirectory(benchmark)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-x86_64.txt
index 033eee4ae6c..64ac719f44e 100644
--- a/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/minikql/jsonpath/CMakeLists.linux-x86_64.txt
@@ -7,6 +7,7 @@
add_subdirectory(benchmark)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/minikql/jsonpath/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/jsonpath/CMakeLists.windows-x86_64.txt
index bb96e932dc2..5224e6130f6 100644
--- a/ydb/library/yql/minikql/jsonpath/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/minikql/jsonpath/CMakeLists.windows-x86_64.txt
@@ -7,6 +7,7 @@
add_subdirectory(benchmark)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..d130b466d35
--- /dev/null
+++ b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,84 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-jsonpath-ut)
+target_compile_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath
+)
+target_link_libraries(ydb-library-yql-minikql-jsonpath-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-jsonpath
+ library-cpp-json
+ ydb-library-binary_json
+ library-yql-minikql
+ minikql-computation-llvm
+ yql-minikql-dom
+ minikql-invoke_builtins-llvm
+ udf-service-exception_policy
+ core-issue-protos
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/examples_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lax_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/strict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/test_base.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lib_id_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-jsonpath-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-jsonpath-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-minikql-jsonpath-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-jsonpath-ut)
diff --git a/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..da30de49141
--- /dev/null
+++ b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,87 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-jsonpath-ut)
+target_compile_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath
+)
+target_link_libraries(ydb-library-yql-minikql-jsonpath-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ yql-minikql-jsonpath
+ library-cpp-json
+ ydb-library-binary_json
+ library-yql-minikql
+ minikql-computation-llvm
+ yql-minikql-dom
+ minikql-invoke_builtins-llvm
+ udf-service-exception_policy
+ core-issue-protos
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/examples_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lax_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/strict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/test_base.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lib_id_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-jsonpath-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-jsonpath-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-minikql-jsonpath-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-minikql-jsonpath-ut)
diff --git a/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..a9532bd37b2
--- /dev/null
+++ b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,89 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-jsonpath-ut)
+target_compile_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath
+)
+target_link_libraries(ydb-library-yql-minikql-jsonpath-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-jsonpath
+ library-cpp-json
+ ydb-library-binary_json
+ library-yql-minikql
+ minikql-computation-llvm
+ yql-minikql-dom
+ minikql-invoke_builtins-llvm
+ udf-service-exception_policy
+ core-issue-protos
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/examples_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lax_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/strict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/test_base.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lib_id_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-jsonpath-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-jsonpath-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-minikql-jsonpath-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-minikql-jsonpath-ut)
diff --git a/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.txt b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..4c482b6b1df
--- /dev/null
+++ b/ydb/library/yql/minikql/jsonpath/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,77 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-minikql-jsonpath-ut)
+target_compile_options(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_include_directories(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath
+)
+target_link_libraries(ydb-library-yql-minikql-jsonpath-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-minikql-jsonpath
+ library-cpp-json
+ ydb-library-binary_json
+ library-yql-minikql
+ minikql-computation-llvm
+ yql-minikql-dom
+ minikql-invoke_builtins-llvm
+ udf-service-exception_policy
+ core-issue-protos
+ yql-sql-pg_dummy
+)
+target_sources(ydb-library-yql-minikql-jsonpath-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/examples_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lax_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/strict_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/test_base.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/jsonpath/ut/lib_id_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-minikql-jsonpath-ut
+ TEST_TARGET
+ ydb-library-yql-minikql-jsonpath-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-minikql-jsonpath-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-minikql-jsonpath-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-minikql-jsonpath-ut)
diff --git a/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp b/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp
index 47a6c68458a..087759f769b 100644
--- a/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp
+++ b/ydb/library/yql/minikql/jsonpath/ut/common_ut.cpp
@@ -339,7 +339,7 @@ public:
"array": [1, 2, 3, 4]
})", "$.array[+$.range.from to +$.range.to]", {"2", "3"}},
{R"([1, 2, 3])", "-$[*]", {"-1", "-2", "-3"}},
- {"10000000000000000000000000", "-$", {"-1e+25"}},
+ {"10000000000000000000000000", "-$", {"-9.999999999999999e+24"}},
};
for (const auto& testCase : testCases) {
diff --git a/ydb/library/yql/minikql/jsonpath/ya.make b/ydb/library/yql/minikql/jsonpath/ya.make
index 629d6246463..68e81ce87e2 100644
--- a/ydb/library/yql/minikql/jsonpath/ya.make
+++ b/ydb/library/yql/minikql/jsonpath/ya.make
@@ -53,6 +53,10 @@ GENERATE_ENUM_SERIALIZATION(ast_nodes.h)
END()
+RECURSE(
+ benchmark
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/minikql/perf/CMakeLists.txt b/ydb/library/yql/minikql/perf/CMakeLists.txt
index 0df202dda16..d9225de1b7d 100644
--- a/ydb/library/yql/minikql/perf/CMakeLists.txt
+++ b/ydb/library/yql/minikql/perf/CMakeLists.txt
@@ -7,6 +7,8 @@
add_subdirectory(alloc)
+add_subdirectory(block_groupby)
+add_subdirectory(mprefetch)
add_subdirectory(mt_param)
add_subdirectory(packer)
add_subdirectory(param)
diff --git a/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..2363b25d571
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,39 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(block_groupby)
+target_compile_options(block_groupby PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(block_groupby PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ libs-apache-arrow
+ yql-minikql-arrow
+ minikql-comp_nodes-llvm
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ library-cpp-getopt
+)
+target_link_options(block_groupby PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(block_groupby PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp
+)
+target_allocator(block_groupby
+ system_allocator
+)
+vcs_info(block_groupby)
diff --git a/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..5990139b8f4
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,42 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(block_groupby)
+target_compile_options(block_groupby PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(block_groupby PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ libs-apache-arrow
+ yql-minikql-arrow
+ minikql-comp_nodes-llvm
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ library-cpp-getopt
+)
+target_link_options(block_groupby PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(block_groupby PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp
+)
+target_allocator(block_groupby
+ cpp-malloc-jemalloc
+)
+vcs_info(block_groupby)
diff --git a/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..6d662c8956c
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,44 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(block_groupby)
+target_compile_options(block_groupby PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(block_groupby PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ libs-apache-arrow
+ yql-minikql-arrow
+ minikql-comp_nodes-llvm
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ library-cpp-getopt
+)
+target_link_options(block_groupby PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(block_groupby PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp
+)
+target_allocator(block_groupby
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(block_groupby)
diff --git a/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.txt b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..e869124c08f
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/block_groupby/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,32 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(block_groupby)
+target_compile_options(block_groupby PRIVATE
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(block_groupby PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ libs-apache-arrow
+ yql-minikql-arrow
+ minikql-comp_nodes-llvm
+ yql-public-udf
+ udf-service-exception_policy
+ yql-sql-pg_dummy
+ library-cpp-getopt
+)
+target_sources(block_groupby PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp
+)
+target_allocator(block_groupby
+ system_allocator
+)
+vcs_info(block_groupby)
diff --git a/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp b/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp
new file mode 100644
index 00000000000..a7ee8e3910d
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/block_groupby/block_groupby.cpp
@@ -0,0 +1,497 @@
+#include <util/datetime/cputimer.h>
+
+#include <ydb/library/yql/minikql/comp_nodes/mkql_rh_hash.h>
+
+#include <ydb/library/yql/minikql/arrow/arrow_defs.h>
+#include <arrow/array/builder_primitive.h>
+#include <arrow/datum.h>
+
+#include <library/cpp/getopt/last_getopt.h>
+#include <util/digest/fnv.h>
+#include <util/digest/murmur.h>
+#include <util/digest/city.h>
+
+enum class EDistribution {
+ Const,
+ Linear,
+ Random,
+ Few,
+ RandomFew
+};
+
+enum class EShape {
+ Default,
+ Sqrt,
+ Log
+};
+
+arrow::Datum MakeIntColumn(ui32 len, EDistribution dist, EShape shape, ui32 buckets) {
+ arrow::Int32Builder builder;
+ ARROW_OK(builder.Reserve(len));
+ for (ui32 i = 0; i < len; ++i) {
+ ui32 val;
+ switch (shape) {
+ case EShape::Default:
+ val = i;
+ break;
+ case EShape::Sqrt:
+ val = (ui32)sqrt(i);
+ break;
+ case EShape::Log:
+ val = (ui32)log(1 + i);
+ break;
+ }
+
+ switch (dist) {
+ case EDistribution::Const:
+ builder.UnsafeAppend(0);
+ break;
+ case EDistribution::Few:
+ builder.UnsafeAppend(val % buckets);
+ break;
+ case EDistribution::Linear:
+ builder.UnsafeAppend(val);
+ break;
+ case EDistribution::Random:
+ builder.UnsafeAppend(IntHash(val));
+ break;
+ case EDistribution::RandomFew:
+ builder.UnsafeAppend(IntHash(val) % buckets);
+ break;
+ }
+ }
+
+ std::shared_ptr<arrow::ArrayData> result;
+ ARROW_OK(builder.FinishInternal(&result));
+ return arrow::Datum(result);
+}
+
+class IAggregator {
+public:
+ virtual ~IAggregator() = default;
+ virtual void Init(i64* state, i32 payload) = 0;
+ virtual void Update(i64* state, i32 payload) = 0;
+};
+
+class TSumAggregator : public IAggregator {
+public:
+ void Init(i64* state, i32 payload) final {
+ *state = payload;
+ }
+
+ void Update(i64* state, i32 payload) final {
+ *state += payload;
+ }
+};
+
+template <typename T>
+struct TCityHasher {
+public:
+ ui64 operator()(const T& x) const {
+ return CityHash64(TStringBuf((char*)&x, sizeof(x)));
+ }
+};
+
+// sum(payloads) group by keys
+template <bool CalculateHashStats, bool UseRH>
+class TAggregate {
+private:
+ struct TOneCell {
+ i32 Key = 0;
+ bool IsEmpty = true;
+ i64 State = 0;
+ };
+
+ struct TCell {
+ i32 Key = 0;
+ i32 PSL = -1;
+ i64 State = 0;
+ };
+
+public:
+ TAggregate(const std::vector<IAggregator*>& aggs)
+ : Aggs(aggs)
+ , RH(sizeof(i64))
+ {
+ Cells.resize(1u << 8);
+ }
+
+ void AddBatch(arrow::Datum keys, arrow::Datum payloads) {
+ auto arrKeys = keys.array();
+ auto arrPayloads = payloads.array();
+ auto len = arrKeys->length;
+ const i32* ptrKeys = arrKeys->GetValues<i32>(1);
+ const i32* ptrPayloads = arrPayloads->GetValues<i32>(1);
+ for (int64_t i = 0; i < len; ++i) {
+ auto key = ptrKeys[i];
+ auto payload = ptrPayloads[i];
+ if (!MoreThanOne) {
+ if (One.IsEmpty) {
+ One.IsEmpty = false;
+ One.Key = key;
+ for (const auto& a : Aggs) {
+ a->Init(&One.State, payload);
+ }
+
+ Size = 1;
+ continue;
+ } else {
+ if (key == One.Key) {
+ for (const auto& a : Aggs) {
+ a->Update(&One.State, payload);
+ }
+
+ continue;
+ } else {
+ MoreThanOne = true;
+ if constexpr (UseRH) {
+ bool isNew;
+ auto iter = RH.Insert(One.Key, isNew);
+ Y_ASSERT(isNew);
+ *(i64*)RH.GetPayload(iter) = One.State;
+ } else {
+ bool isNew;
+ ui64 bucket = AddBucketFromKeyImpl(One.Key, Cells, isNew);
+ auto& c = Cells[bucket];
+ c.PSL = 0;
+ c.Key = One.Key;
+ c.State = One.State;
+ }
+ }
+ }
+ }
+
+ if constexpr (UseRH) {
+ bool isNew = false;
+ auto iter = RH.Insert(key, isNew);
+ if (isNew) {
+ for (const auto& a : Aggs) {
+ a->Init((i64*)RH.GetPayload(iter), payload);
+ }
+
+ RH.CheckGrow();
+ } else {
+ for (const auto& a : Aggs) {
+ a->Update((i64*)RH.GetPayload(iter), payload);
+ }
+ }
+ } else {
+ bool isNew = false;
+ ui64 bucket = AddBucketFromKey(key, isNew);
+ auto& c = Cells[bucket];
+ if (isNew) {
+ Size += 1;
+ for (const auto& a : Aggs) {
+ a->Init(&c.State, payload);
+ }
+
+ if (Size * 2 >= Cells.size()) {
+ Grow();
+ }
+ } else {
+ for (const auto& a : Aggs) {
+ a->Update(&c.State, payload);
+ }
+ }
+ }
+ }
+ }
+
+ static ui64 MakeHash(i32 key) {
+ //auto hash = FnvHash<ui64>(&key, sizeof(key));
+ //auto hash = MurmurHash<ui64>(&key, sizeof(key));
+ auto hash = CityHash64(TStringBuf((char*)&key, sizeof(key)));
+ //auto hash = key;
+ return hash;
+ }
+
+ Y_FORCE_INLINE ui64 AddBucketFromKey(i32 key, bool& isNew) {
+ return AddBucketFromKeyImpl(key, Cells, isNew);
+ }
+
+ Y_FORCE_INLINE ui64 AddBucketFromKeyImpl(i32 key, std::vector<TCell>& cells, bool& isNew) {
+ isNew = false;
+ ui32 chainLen = 0;
+ if constexpr (CalculateHashStats) {
+ HashSearches++;
+ }
+
+ ui64 bucket = MakeHash(key) & (cells.size() - 1);
+ i32 distance = 0;
+ ui64 returnBucket;
+ i64 oldState;
+ for (;;) {
+ if constexpr (CalculateHashStats) {
+ HashProbes++;
+ chainLen++;
+ }
+
+ if (cells[bucket].PSL < 0) {
+ isNew = true;
+ cells[bucket].Key = key;
+ cells[bucket].PSL = distance;
+
+ if constexpr (CalculateHashStats) {
+ MaxHashChainLen = Max(MaxHashChainLen, chainLen);
+ }
+
+ return bucket;
+ }
+
+ if (cells[bucket].Key == key) {
+ if constexpr (CalculateHashStats) {
+ MaxHashChainLen = Max(MaxHashChainLen, chainLen);
+ }
+
+ return bucket;
+ }
+
+ if (distance > cells[bucket].PSL) {
+ // swap keys & state
+ returnBucket = bucket;
+ oldState = cells[bucket].State;
+ std::swap(key, cells[bucket].Key);
+ std::swap(distance, cells[bucket].PSL);
+ isNew = true;
+
+ ++distance;
+ bucket = (bucket + 1) & (cells.size() - 1);
+ break;
+ }
+
+ ++distance;
+ bucket = (bucket + 1) & (cells.size() - 1);
+ }
+
+ for (;;) {
+ if constexpr (CalculateHashStats) {
+ HashProbes++;
+ chainLen++;
+ }
+
+ if (cells[bucket].PSL < 0) {
+ if constexpr (CalculateHashStats) {
+ MaxHashChainLen = Max(MaxHashChainLen, chainLen);
+ }
+
+ cells[bucket].Key = key;
+ cells[bucket].State = oldState;
+ cells[bucket].PSL = distance;
+ return returnBucket; // for original key
+ }
+
+ Y_ENSURE(cells[bucket].Key != key);
+ if (distance > cells[bucket].PSL) {
+ // swap keys & state
+ std::swap(key, cells[bucket].Key);
+ std::swap(oldState, cells[bucket].State);
+ std::swap(distance, cells[bucket].PSL);
+ }
+
+ ++distance;
+ bucket = (bucket + 1) & (cells.size() - 1);
+ }
+ }
+
+ void Grow() {
+ std::vector<TCell> newCells;
+ newCells.resize(Cells.size() * 2); // must be power of 2
+ for (const auto& c : Cells) {
+ if (c.PSL < 0) {
+ continue;
+ }
+
+ bool isNew;
+ auto newBucket = AddBucketFromKeyImpl(c.Key, newCells, isNew);
+ auto& nc = newCells[newBucket];
+ nc.State = c.State;
+ }
+
+ Cells.swap(newCells);
+ }
+
+ double GetAverageHashChainLen() {
+ return 1.0*HashProbes/HashSearches;
+ }
+
+ ui32 GetMaxHashChainLen() {
+ return MaxHashChainLen;
+ }
+
+ void GetResult(arrow::Datum& keys, arrow::Datum& sums) {
+ arrow::Int32Builder keysBuilder;
+ arrow::Int64Builder sumsBuilder;
+ if (!MoreThanOne) {
+ if (!One.IsEmpty) {
+ ARROW_OK(keysBuilder.Reserve(1));
+ ARROW_OK(sumsBuilder.Reserve(1));
+ keysBuilder.UnsafeAppend(One.Key);
+ sumsBuilder.UnsafeAppend(One.State);
+ }
+ } else {
+ ui64 size;
+ if constexpr (UseRH) {
+ size = RH.GetSize();
+ } else {
+ size = Size;
+ }
+
+ ARROW_OK(keysBuilder.Reserve(size));
+ ARROW_OK(sumsBuilder.Reserve(size));
+ i32 maxPSL = 0;
+ i64 sumPSL = 0;
+ if constexpr (UseRH) {
+ for (auto iter = RH.Begin(); iter != RH.End(); RH.Advance(iter)) {
+ auto& psl = RH.GetPSL(iter);
+ if (psl.Distance < 0) {
+ continue;
+ }
+
+ keysBuilder.UnsafeAppend(RH.GetKey(iter));
+ sumsBuilder.UnsafeAppend(*(i64*)RH.GetPayload(iter));
+ maxPSL = Max(psl.Distance, maxPSL);
+ sumPSL += psl.Distance;
+ }
+ } else {
+ for (const auto& c : Cells) {
+ if (c.PSL < 0) {
+ continue;
+ }
+
+ keysBuilder.UnsafeAppend(c.Key);
+ sumsBuilder.UnsafeAppend(c.State);
+ maxPSL = Max(c.PSL, maxPSL);
+ sumPSL += c.PSL;
+ }
+ }
+
+ if constexpr (CalculateHashStats) {
+ Cerr << "maxPSL = " << maxPSL << "\n";
+ Cerr << "avgPSL = " << 1.0*sumPSL/size << "\n";
+ }
+ }
+
+ std::shared_ptr<arrow::ArrayData> keysData;
+ ARROW_OK(keysBuilder.FinishInternal(&keysData));
+ keys = keysData;
+
+ std::shared_ptr<arrow::ArrayData> sumsData;
+ ARROW_OK(sumsBuilder.FinishInternal(&sumsData));
+ sums = sumsData;
+ }
+
+private:
+ bool MoreThanOne = false;
+ TOneCell One;
+ std::vector<TCell> Cells;
+ ui64 Size = 0;
+
+ const std::vector<IAggregator*> Aggs;
+ ui64 HashProbes = 0;
+ ui64 HashSearches = 0;
+ ui32 MaxHashChainLen = 0;
+
+ NKikimr::NMiniKQL::TRobinHoodHashMap<i32> RH;
+ NKikimr::NMiniKQL::TRobinHoodHashSet<i32> RHS;
+};
+
+int main(int argc, char** argv) {
+ NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default();
+ TString keysDistributionStr;
+ TString shapeStr="default";
+ ui32 nIters = 100;
+ ui32 nRows = 1000000;
+ ui32 nBuckets = 16;
+ ui32 nRepeats = 10;
+ opts.AddLongOption('k', "keys", "distribution of keys (const, linear, random, few, randomfew)").StoreResult(&keysDistributionStr).Required();
+ opts.AddLongOption('s', "shape", "shape of counter (default, sqrt, log)").StoreResult(&shapeStr);
+ opts.AddLongOption('i', "iter", "# of iterations").StoreResult(&nIters);
+ opts.AddLongOption('r', "rows", "# of rows").StoreResult(&nRows);
+ opts.AddLongOption('b', "buckets", "modulo for few/randomfew").StoreResult(&nBuckets);
+ opts.AddLongOption('t', "repeats", "# of repeats").StoreResult(&nRepeats);
+ opts.SetFreeArgsMax(0);
+ NLastGetopt::TOptsParseResult res(&opts, argc, argv);
+ EDistribution keysDist;
+ EShape shape = EShape::Default;
+ if (keysDistributionStr == "const") {
+ keysDist = EDistribution::Const;
+ } else if (keysDistributionStr == "linear") {
+ keysDist = EDistribution::Linear;
+ } else if (keysDistributionStr == "random") {
+ keysDist = EDistribution::Random;
+ } else if (keysDistributionStr == "few") {
+ keysDist = EDistribution::Few;
+ } else if (keysDistributionStr == "randomfew") {
+ keysDist = EDistribution::RandomFew;
+ } else {
+ ythrow yexception() << "Unsupported distribution: " << keysDistributionStr;
+ }
+
+ if (shapeStr == "default") {
+ shape = EShape::Default;
+ } else if (shapeStr == "sqrt") {
+ shape = EShape::Sqrt;
+ } else if (shapeStr == "log") {
+ shape = EShape::Log;
+ } else {
+ ythrow yexception() << "Unsupported shape: " << shapeStr;
+ }
+
+ auto col1 = MakeIntColumn(nRows, keysDist, shape, nBuckets);
+ auto col2 = MakeIntColumn(nRows, EDistribution::Linear, EShape::Default, nBuckets);
+ Cerr << "col1.length: " << col1.length() << "\n";
+ Cerr << "col2.length: " << col2.length() << "\n";
+
+ TSumAggregator sum;
+ std::vector<IAggregator*> aggs;
+ aggs.push_back(&sum);
+ TAggregate<true, true> agg(aggs);
+ agg.AddBatch(col1, col2);
+ arrow::Datum keys, sums;
+ agg.GetResult(keys, sums);
+ ui64 total1 = 0;
+ for (ui32 i = 0; i < col2.length(); ++i) {
+ total1 += col2.array()->GetValues<i32>(1)[i];
+ }
+
+ Cerr << "total1: " << total1 << "\n";
+ ui64 total2 = 0;
+
+ Cerr << "keys.length: " << keys.length() << "\n";
+ Cerr << "sums.length: " << sums.length() << "\n";
+ for (ui32 i = 0; i < sums.length(); ++i) {
+ total2 += sums.array()->GetValues<i64>(1)[i];
+ }
+
+ Cerr << "total2: " << total2 << "\n";
+ Y_ENSURE(total1 == total2);
+ Cerr << "AverageHashChainLen: " << agg.GetAverageHashChainLen() << "\n";
+ Cerr << "MaxHashChainLen: " << agg.GetMaxHashChainLen() << "\n";
+
+ std::vector<double> durations;
+ for (ui32 j = 0; j < nRepeats; ++j) {
+ TSimpleTimer timer;
+ for (ui32 i = 0; i < nIters; ++i) {
+ TAggregate<false, true> agg(aggs);
+ agg.AddBatch(col1, col2);
+ arrow::Datum keys, sums;
+ agg.GetResult(keys, sums);
+ }
+
+ auto duration = timer.Get();
+ durations.push_back(1e-6*duration.MicroSeconds());
+ }
+
+ double sumDurations = 0.0, sumDurationsQ = 0.0;
+ for (auto d : durations) {
+ sumDurations += d;
+ sumDurationsQ += d * d;
+ }
+
+ double avgDuration = sumDurations / nRepeats;
+ double dispDuration = sqrt(sumDurationsQ / nRepeats - avgDuration * avgDuration);
+ Cerr << "Elapsed: " << avgDuration << ", noise: " << 100*dispDuration/avgDuration << "%\n";
+ Cerr << "Speed: " << 1e-6 * (ui64(nIters) * nRows / avgDuration) << " M rows/sec\n";
+ Cerr << "Speed: " << 1e-6 * (2 * sizeof(i32) * ui64(nIters) * nRows / avgDuration) << " M bytes/sec\n";
+ return 0;
+}
diff --git a/ydb/library/yql/minikql/perf/block_groupby/ya.make b/ydb/library/yql/minikql/perf/block_groupby/ya.make
new file mode 100644
index 00000000000..29e7fd95ca1
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/block_groupby/ya.make
@@ -0,0 +1,19 @@
+PROGRAM()
+
+PEERDIR(
+ contrib/libs/apache/arrow
+ ydb/library/yql/minikql/arrow
+ ydb/library/yql/minikql/comp_nodes/llvm
+ ydb/library/yql/public/udf
+ ydb/library/yql/public/udf/service/exception_policy
+ ydb/library/yql/sql/pg_dummy
+ library/cpp/getopt
+)
+
+SRCS(
+ block_groupby.cpp
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..d0b1af3792f
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,32 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(mprefetch)
+target_compile_options(mprefetch PRIVATE
+ -mprfchw
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(mprefetch PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ library-cpp-getopt
+)
+target_link_options(mprefetch PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+)
+target_sources(mprefetch PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp
+)
+target_allocator(mprefetch
+ system_allocator
+)
+vcs_info(mprefetch)
diff --git a/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-aarch64.txt b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..318241f220e
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,37 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(mprefetch)
+target_compile_options(mprefetch PRIVATE
+ -mprfchw
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(mprefetch PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-getopt
+)
+target_link_options(mprefetch PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(mprefetch PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp
+)
+target_allocator(mprefetch
+ cpp-malloc-jemalloc
+)
+vcs_info(mprefetch)
diff --git a/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-x86_64.txt b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..bd71fd7c81f
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,39 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(mprefetch)
+target_compile_options(mprefetch PRIVATE
+ -mprfchw
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(mprefetch PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ library-cpp-getopt
+)
+target_link_options(mprefetch PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(mprefetch PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp
+)
+target_allocator(mprefetch
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(mprefetch)
diff --git a/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.txt b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.windows-x86_64.txt b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..f2f199b2400
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/mprefetch/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,27 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(mprefetch)
+target_compile_options(mprefetch PRIVATE
+ -mprfchw
+ -DUSE_CURRENT_UDF_ABI_VERSION
+)
+target_link_libraries(mprefetch PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ library-cpp-getopt
+)
+target_sources(mprefetch PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp
+)
+target_allocator(mprefetch
+ system_allocator
+)
+vcs_info(mprefetch)
diff --git a/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp b/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp
new file mode 100644
index 00000000000..9ba9cbc3df4
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/mprefetch/mprefetch.cpp
@@ -0,0 +1,108 @@
+#include <util/datetime/cputimer.h>
+
+#include <library/cpp/getopt/last_getopt.h>
+
+#include <deque>
+
+int main(int argc, char** argv) {
+ NLastGetopt::TOpts opts = NLastGetopt::TOpts::Default();
+ ui32 nIters = 100000000;
+ ui32 nRows = 100000000;
+ ui32 nRepeats = 10;
+ ui32 nPrefetch = 0;
+ ui32 nSpin = 0;
+ bool shuffle = true;
+ opts.AddLongOption('r', "rows", "# of rows").StoreResult(&nRows);
+ opts.AddLongOption('i', "iter", "# of iterations").StoreResult(&nIters);
+ opts.AddLongOption('t', "repeats", "# of repeats").StoreResult(&nRepeats);
+ opts.AddLongOption('p', "prefetch", "# of prefetch").StoreResult(&nPrefetch);
+ opts.AddLongOption('h', "shuffle", "randomize").StoreResult(&shuffle);
+ opts.AddLongOption('s', "spin", "spin count").StoreResult(&nSpin);
+ opts.SetFreeArgsMax(0);
+ NLastGetopt::TOptsParseResult res(&opts, argc, argv);
+
+ std::vector<ui32> v(nRows);
+ std::vector<ui32> data(nRows);
+ std::iota(v.begin(), v.end(), 0);
+ if (shuffle) {
+ std::random_shuffle(v.begin(), v.end());
+ }
+
+ std::vector<ui32> prefetchQueue(nPrefetch);
+ ui32 queueBegin = 0;
+ ui32 queueEnd = 0;
+ ui32 queueSize = 0;
+ volatile ui64 tmp = 0;
+ std::vector<double> durations;
+ for (ui32 j = 0; j < nRepeats; ++j) {
+ TSimpleTimer timer;
+ ui32 index = 0;
+ if (nPrefetch == 0) {
+ for (ui32 i = 0; i < nIters; ++i) {
+ data[v[index++]]+=1;
+ if (index == nRows) {
+ index = 0;
+ }
+
+ for (ui32 j = 0; j < nSpin; ++j) {
+ ++tmp;
+ }
+ }
+ } else {
+ auto handle = [&]() {
+ auto prevJ = prefetchQueue[queueBegin++];
+ --queueSize;
+ if (queueBegin == nPrefetch) {
+ queueBegin = 0;
+ }
+
+ data[prevJ]+=1;
+
+ for (ui32 j = 0; j < nSpin; ++j) {
+ ++tmp;
+ }
+ };
+
+ for (ui32 i = 0; i < nIters; ++i) {
+ auto j = v[index++];
+ if (index == nRows) {
+ index = 0;
+ }
+
+ __builtin_prefetch(data.data() + j, 1, 3);
+ prefetchQueue[queueEnd++] = j;
+ ++queueSize;
+ if (queueEnd == nPrefetch) {
+ queueEnd = 0;
+ }
+
+ if (queueSize == nPrefetch) {
+ handle();
+ }
+ }
+
+ while (queueSize > 0) {
+ handle();
+ }
+ }
+
+ auto duration = timer.Get();
+ durations.push_back(1e-6*duration.MicroSeconds());
+ }
+
+ // remove 1/3 of worst measurements
+ Sort(durations.begin(), durations.end());
+ durations.erase(durations.begin() + nRepeats * 2 / 3, durations.end());
+ nRepeats = durations.size();
+ double sumDurations = 0.0, sumDurationsQ = 0.0;
+ for (auto d : durations) {
+ sumDurations += d;
+ sumDurationsQ += d * d;
+ }
+
+ double avgDuration = sumDurations / nRepeats;
+ double dispDuration = sqrt(sumDurationsQ / nRepeats - avgDuration * avgDuration);
+ Cerr << "Elapsed: " << avgDuration << ", noise: " << 100*dispDuration/avgDuration << "%\n";
+ Cerr << "Speed: " << 1e-6 * (ui64(nIters) / avgDuration) << " M iters/sec\n";
+ return 0;
+}
diff --git a/ydb/library/yql/minikql/perf/mprefetch/ya.make b/ydb/library/yql/minikql/perf/mprefetch/ya.make
new file mode 100644
index 00000000000..c7d873f7237
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/mprefetch/ya.make
@@ -0,0 +1,17 @@
+PROGRAM()
+
+PEERDIR(
+ library/cpp/getopt
+)
+
+SRCS(
+ mprefetch.cpp
+)
+
+CFLAGS(
+ -mprfchw
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/minikql/perf/ya.make b/ydb/library/yql/minikql/perf/ya.make
new file mode 100644
index 00000000000..dd83a764f9a
--- /dev/null
+++ b/ydb/library/yql/minikql/perf/ya.make
@@ -0,0 +1,10 @@
+
+RECURSE(
+ alloc
+ block_groupby
+ mprefetch
+ mt_param
+ packer
+ param
+ presort
+)
diff --git a/ydb/library/yql/minikql/ya.make b/ydb/library/yql/minikql/ya.make
index fd1ef197f08..8c64d9eb3c7 100644
--- a/ydb/library/yql/minikql/ya.make
+++ b/ydb/library/yql/minikql/ya.make
@@ -83,6 +83,19 @@ YQL_LAST_ABI_VERSION()
END()
+RECURSE(
+ arrow
+ benchmark
+ codegen
+ comp_nodes
+ computation
+ datetime
+ dom
+ invoke_builtins
+ jsonpath
+ perf
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/parser/proto_ast/ya.make b/ydb/library/yql/parser/proto_ast/ya.make
index fb7290cadb5..8251c29c182 100644
--- a/ydb/library/yql/parser/proto_ast/ya.make
+++ b/ydb/library/yql/parser/proto_ast/ya.make
@@ -12,6 +12,6 @@ SRCS(
END()
RECURSE(
- gen
collect_issues
+ gen
)
diff --git a/ydb/library/yql/providers/clickhouse/ya.make b/ydb/library/yql/providers/clickhouse/ya.make
index eacab82215a..58d9016b198 100644
--- a/ydb/library/yql/providers/clickhouse/ya.make
+++ b/ydb/library/yql/providers/clickhouse/ya.make
@@ -1,6 +1,6 @@
RECURSE(
actors
expr_nodes
- provider
proto
+ provider
)
diff --git a/ydb/library/yql/providers/common/arrow/ya.make b/ydb/library/yql/providers/common/arrow/ya.make
index 5c032323097..328794fdd57 100644
--- a/ydb/library/yql/providers/common/arrow/ya.make
+++ b/ydb/library/yql/providers/common/arrow/ya.make
@@ -14,4 +14,8 @@ CFLAGS(
-DARCADIA_BUILD -DUSE_PARQUET
)
-END() \ No newline at end of file
+END()
+
+RECURSE(
+ interface
+)
diff --git a/ydb/library/yql/providers/common/metrics/ya.make b/ydb/library/yql/providers/common/metrics/ya.make
index c40f275be35..8b54fc73a0d 100644
--- a/ydb/library/yql/providers/common/metrics/ya.make
+++ b/ydb/library/yql/providers/common/metrics/ya.make
@@ -13,3 +13,7 @@ PEERDIR(
)
END()
+
+RECURSE(
+ protos
+)
diff --git a/ydb/library/yql/providers/common/proto/python/ya.make b/ydb/library/yql/providers/common/proto/python/ya.make
new file mode 100644
index 00000000000..fc9cc6f5b91
--- /dev/null
+++ b/ydb/library/yql/providers/common/proto/python/ya.make
@@ -0,0 +1,2 @@
+PY_PROTOS_FOR(ydb/library/yql/providers/common/proto)
+
diff --git a/ydb/library/yql/providers/common/schema/ya.make b/ydb/library/yql/providers/common/schema/ya.make
index 4357f787d5d..f807ad3fe26 100644
--- a/ydb/library/yql/providers/common/schema/ya.make
+++ b/ydb/library/yql/providers/common/schema/ya.make
@@ -10,3 +10,10 @@ PEERDIR(
)
END()
+
+RECURSE(
+ expr
+ mkql
+ parser
+ skiff
+)
diff --git a/ydb/library/yql/providers/common/ya.make b/ydb/library/yql/providers/common/ya.make
index 275a09ad198..7bef7eaedda 100644
--- a/ydb/library/yql/providers/common/ya.make
+++ b/ydb/library/yql/providers/common/ya.make
@@ -1,8 +1,9 @@
RECURSE(
activation
- arrow_resolve
arrow
+ arrow_resolve
codec
+ comp_nodes
config
db_id_async_resolver
dq
@@ -10,14 +11,10 @@ RECURSE(
http_gateway
metrics
mkql
- comp_nodes
proto
+ proto/python
provider
schema
- schema/expr
- schema/mkql
- schema/parser
- schema/skiff
structured_token
token_accessor
transform
diff --git a/ydb/library/yql/providers/dq/provider/ya.make b/ydb/library/yql/providers/dq/provider/ya.make
index 5765c684d44..aec22bc2bdc 100644
--- a/ydb/library/yql/providers/dq/provider/ya.make
+++ b/ydb/library/yql/providers/dq/provider/ya.make
@@ -61,6 +61,10 @@ YQL_LAST_ABI_VERSION()
END()
+RECURSE(
+ exec
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/providers/dq/worker_manager/ya.make b/ydb/library/yql/providers/dq/worker_manager/ya.make
index 63c61eabd8d..8ef4840b723 100644
--- a/ydb/library/yql/providers/dq/worker_manager/ya.make
+++ b/ydb/library/yql/providers/dq/worker_manager/ya.make
@@ -22,3 +22,7 @@ SRCS(
)
END()
+
+RECURSE(
+ interface
+)
diff --git a/ydb/library/yql/providers/dq/ya.make b/ydb/library/yql/providers/dq/ya.make
index f4af5913426..e354a8a22f4 100644
--- a/ydb/library/yql/providers/dq/ya.make
+++ b/ydb/library/yql/providers/dq/ya.make
@@ -10,7 +10,6 @@ RECURSE(
opt
planner
provider
- provider/exec
runtime
task_runner
task_runner_actor
diff --git a/ydb/library/yql/providers/generic/ya.make b/ydb/library/yql/providers/generic/ya.make
index 26a0d87ba62..5e9adf2bc83 100644
--- a/ydb/library/yql/providers/generic/ya.make
+++ b/ydb/library/yql/providers/generic/ya.make
@@ -2,8 +2,8 @@ RECURSE(
actors
connector
expr_nodes
- provider
proto
+ provider
)
RECURSE_FOR_TESTS(
diff --git a/ydb/library/yql/providers/s3/object_listers/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/CMakeLists.darwin-x86_64.txt
index 5b00b830ac8..ad94789cb73 100644
--- a/ydb/library/yql/providers/s3/object_listers/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/providers/s3/object_listers/CMakeLists.darwin-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-aarch64.txt
index 0155191e4bf..abf0a0e4d69 100644
--- a/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-aarch64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-x86_64.txt
index 0155191e4bf..abf0a0e4d69 100644
--- a/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/providers/s3/object_listers/CMakeLists.linux-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/providers/s3/object_listers/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/CMakeLists.windows-x86_64.txt
index 5b00b830ac8..ad94789cb73 100644
--- a/ydb/library/yql/providers/s3/object_listers/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/providers/s3/object_listers/CMakeLists.windows-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..8cb595c1e1c
--- /dev/null
+++ b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,67 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-providers-s3-object_listers-ut)
+target_include_directories(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers
+)
+target_link_libraries(ydb-library-yql-providers-s3-object_listers-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ providers-s3-object_listers
+)
+target_link_options(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers/yql_s3_path_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-providers-s3-object_listers-ut
+ TEST_TARGET
+ ydb-library-yql-providers-s3-object_listers-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-providers-s3-object_listers-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-providers-s3-object_listers-ut)
diff --git a/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..401b4507243
--- /dev/null
+++ b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,70 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-providers-s3-object_listers-ut)
+target_include_directories(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers
+)
+target_link_libraries(ydb-library-yql-providers-s3-object_listers-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ providers-s3-object_listers
+)
+target_link_options(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers/yql_s3_path_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-providers-s3-object_listers-ut
+ TEST_TARGET
+ ydb-library-yql-providers-s3-object_listers-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-providers-s3-object_listers-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-providers-s3-object_listers-ut)
diff --git a/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..cbb021028ba
--- /dev/null
+++ b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,72 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-providers-s3-object_listers-ut)
+target_include_directories(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers
+)
+target_link_libraries(ydb-library-yql-providers-s3-object_listers-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ providers-s3-object_listers
+)
+target_link_options(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers/yql_s3_path_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-providers-s3-object_listers-ut
+ TEST_TARGET
+ ydb-library-yql-providers-s3-object_listers-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-providers-s3-object_listers-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-providers-s3-object_listers-ut)
diff --git a/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.txt b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..eeeb92a6ea2
--- /dev/null
+++ b/ydb/library/yql/providers/s3/object_listers/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,60 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-providers-s3-object_listers-ut)
+target_include_directories(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers
+)
+target_link_libraries(ydb-library-yql-providers-s3-object_listers-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ providers-s3-object_listers
+)
+target_sources(ydb-library-yql-providers-s3-object_listers-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/providers/s3/object_listers/yql_s3_path_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-providers-s3-object_listers-ut
+ TEST_TARGET
+ ydb-library-yql-providers-s3-object_listers-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-providers-s3-object_listers-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-providers-s3-object_listers-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-providers-s3-object_listers-ut)
diff --git a/ydb/library/yql/providers/s3/ya.make b/ydb/library/yql/providers/s3/ya.make
index 6408970d3b3..0f22cf790ee 100644
--- a/ydb/library/yql/providers/s3/ya.make
+++ b/ydb/library/yql/providers/s3/ya.make
@@ -3,6 +3,7 @@ RECURSE(
common
compressors
expr_nodes
+ object_listers
path_generator
proto
provider
diff --git a/ydb/library/yql/providers/stat/ya.make b/ydb/library/yql/providers/stat/ya.make
new file mode 100644
index 00000000000..fc76f599d96
--- /dev/null
+++ b/ydb/library/yql/providers/stat/ya.make
@@ -0,0 +1,5 @@
+RECURSE(
+ expr_nodes
+ uploader
+)
+
diff --git a/ydb/library/yql/providers/ya.make b/ydb/library/yql/providers/ya.make
index 5366e2c7d8f..bae71fcd1e6 100644
--- a/ydb/library/yql/providers/ya.make
+++ b/ydb/library/yql/providers/ya.make
@@ -3,13 +3,14 @@ RECURSE(
common
config
dq
+ function
+ generic
pq
result
s3
solomon
+ stat
ydb
yt
- function
- generic
)
diff --git a/ydb/library/yql/providers/yt/codec/ya.make b/ydb/library/yql/providers/yt/codec/ya.make
index ce64d963adf..558538625b4 100644
--- a/ydb/library/yql/providers/yt/codec/ya.make
+++ b/ydb/library/yql/providers/yt/codec/ya.make
@@ -40,6 +40,10 @@ YQL_LAST_ABI_VERSION()
END()
+RECURSE(
+ codegen
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/providers/yt/comp_nodes/ya.make b/ydb/library/yql/providers/yt/comp_nodes/ya.make
index b1d3159abd2..5b950ff88b7 100644
--- a/ydb/library/yql/providers/yt/comp_nodes/ya.make
+++ b/ydb/library/yql/providers/yt/comp_nodes/ya.make
@@ -25,6 +25,10 @@ YQL_LAST_ABI_VERSION()
END()
+RECURSE(
+ dq
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/providers/yt/lib/ya.make b/ydb/library/yql/providers/yt/lib/ya.make
index 9fb3a47e1c9..108cc9d99bd 100644
--- a/ydb/library/yql/providers/yt/lib/ya.make
+++ b/ydb/library/yql/providers/yt/lib/ya.make
@@ -1,4 +1,5 @@
RECURSE(
+ config_clusters
expr_traits
graph_reorder
hash
@@ -14,6 +15,6 @@ RECURSE(
skiff
url_mapper
yson_helpers
- yt_url_lister
yt_download
+ yt_url_lister
)
diff --git a/ydb/library/yql/providers/yt/ya.make b/ydb/library/yql/providers/yt/ya.make
index a43252062fa..1a788e63871 100644
--- a/ydb/library/yql/providers/yt/ya.make
+++ b/ydb/library/yql/providers/yt/ya.make
@@ -1,11 +1,7 @@
RECURSE(
codec
- codec/codegen/ut
- codec/ut
common
comp_nodes
- comp_nodes/ut
- comp_nodes/dq
expr_nodes
gateway
job
@@ -13,5 +9,4 @@ RECURSE(
mkql_dq
opt
provider
- provider/ut
)
diff --git a/ydb/library/yql/public/issue/ya.make b/ydb/library/yql/public/issue/ya.make
index 65441b9531c..8d7ad13b1c6 100644
--- a/ydb/library/yql/public/issue/ya.make
+++ b/ydb/library/yql/public/issue/ya.make
@@ -21,6 +21,10 @@ GENERATE_ENUM_SERIALIZATION(yql_warning.h)
END()
+RECURSE(
+ protos
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/public/purecalc/examples/protobuf_pull_list/ya.make b/ydb/library/yql/public/purecalc/examples/protobuf_pull_list/ya.make
index 953ff1bf921..cf800933af6 100644
--- a/ydb/library/yql/public/purecalc/examples/protobuf_pull_list/ya.make
+++ b/ydb/library/yql/public/purecalc/examples/protobuf_pull_list/ya.make
@@ -11,9 +11,7 @@ PEERDIR(
ydb/library/yql/public/purecalc/helpers/stream
)
-
- YQL_LAST_ABI_VERSION()
-
+YQL_LAST_ABI_VERSION()
END()
diff --git a/ydb/library/yql/public/purecalc/examples/ya.make b/ydb/library/yql/public/purecalc/examples/ya.make
index ad5853e9c49..d78f8a825d0 100644
--- a/ydb/library/yql/public/purecalc/examples/ya.make
+++ b/ydb/library/yql/public/purecalc/examples/ya.make
@@ -1,7 +1,5 @@
RECURSE(
protobuf
- protobuf/ut
protobuf_pull_list
- protobuf_pull_list/ut
skiff_pull_list
)
diff --git a/ydb/library/yql/public/purecalc/io_specs/protobuf/ya.make b/ydb/library/yql/public/purecalc/io_specs/protobuf/ya.make
index a9efad989f5..7177024003f 100644
--- a/ydb/library/yql/public/purecalc/io_specs/protobuf/ya.make
+++ b/ydb/library/yql/public/purecalc/io_specs/protobuf/ya.make
@@ -10,9 +10,7 @@ SRCS(
proto_variant.cpp
)
-
- YQL_LAST_ABI_VERSION()
-
+YQL_LAST_ABI_VERSION()
END()
diff --git a/ydb/library/yql/public/purecalc/ya.make b/ydb/library/yql/public/purecalc/ya.make
index 3ac6ab6b59a..cb99c1d3111 100644
--- a/ydb/library/yql/public/purecalc/ya.make
+++ b/ydb/library/yql/public/purecalc/ya.make
@@ -13,7 +13,13 @@ YQL_LAST_ABI_VERSION()
END()
+RECURSE(
+ common
+ examples
+ helpers
+ io_specs
+)
+
RECURSE_FOR_TESTS(
- io_specs/ut
ut
)
diff --git a/ydb/library/yql/public/udf/service/ya.make b/ydb/library/yql/public/udf/service/ya.make
new file mode 100644
index 00000000000..e03927ed91b
--- /dev/null
+++ b/ydb/library/yql/public/udf/service/ya.make
@@ -0,0 +1,5 @@
+RECURSE(
+ exception_policy
+ stub
+ terminate_policy
+)
diff --git a/ydb/library/yql/public/udf/tz/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/public/udf/tz/CMakeLists.darwin-x86_64.txt
index 34a38599296..764f26f8b15 100644
--- a/ydb/library/yql/public/udf/tz/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/public/udf/tz/CMakeLists.darwin-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(public-udf-tz)
target_link_libraries(public-udf-tz PUBLIC
diff --git a/ydb/library/yql/public/udf/tz/CMakeLists.linux-aarch64.txt b/ydb/library/yql/public/udf/tz/CMakeLists.linux-aarch64.txt
index 324920a3a14..85f78011a68 100644
--- a/ydb/library/yql/public/udf/tz/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/public/udf/tz/CMakeLists.linux-aarch64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(public-udf-tz)
target_link_libraries(public-udf-tz PUBLIC
diff --git a/ydb/library/yql/public/udf/tz/CMakeLists.linux-x86_64.txt b/ydb/library/yql/public/udf/tz/CMakeLists.linux-x86_64.txt
index 324920a3a14..85f78011a68 100644
--- a/ydb/library/yql/public/udf/tz/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/public/udf/tz/CMakeLists.linux-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(public-udf-tz)
target_link_libraries(public-udf-tz PUBLIC
diff --git a/ydb/library/yql/public/udf/tz/CMakeLists.windows-x86_64.txt b/ydb/library/yql/public/udf/tz/CMakeLists.windows-x86_64.txt
index 34a38599296..764f26f8b15 100644
--- a/ydb/library/yql/public/udf/tz/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/public/udf/tz/CMakeLists.windows-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
add_library(public-udf-tz)
target_link_libraries(public-udf-tz PUBLIC
diff --git a/ydb/library/yql/public/udf/tz/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/public/udf/tz/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..a26f52dd986
--- /dev/null
+++ b/ydb/library/yql/public/udf/tz/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,65 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-public-udf-tz-ut)
+target_include_directories(ydb-library-yql-public-udf-tz-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz
+)
+target_link_libraries(ydb-library-yql-public-udf-tz-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ public-udf-tz
+)
+target_link_options(ydb-library-yql-public-udf-tz-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+)
+target_sources(ydb-library-yql-public-udf-tz-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz/udf_tz_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-public-udf-tz-ut
+ TEST_TARGET
+ ydb-library-yql-public-udf-tz-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-public-udf-tz-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-public-udf-tz-ut)
diff --git a/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..de01319ee97
--- /dev/null
+++ b/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,70 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-public-udf-tz-ut)
+target_include_directories(ydb-library-yql-public-udf-tz-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz
+)
+target_link_libraries(ydb-library-yql-public-udf-tz-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ public-udf-tz
+)
+target_link_options(ydb-library-yql-public-udf-tz-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-public-udf-tz-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz/udf_tz_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-public-udf-tz-ut
+ TEST_TARGET
+ ydb-library-yql-public-udf-tz-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-public-udf-tz-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-public-udf-tz-ut)
diff --git a/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..d7f98a2a6c2
--- /dev/null
+++ b/ydb/library/yql/public/udf/tz/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,72 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-public-udf-tz-ut)
+target_include_directories(ydb-library-yql-public-udf-tz-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz
+)
+target_link_libraries(ydb-library-yql-public-udf-tz-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ public-udf-tz
+)
+target_link_options(ydb-library-yql-public-udf-tz-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-public-udf-tz-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz/udf_tz_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-public-udf-tz-ut
+ TEST_TARGET
+ ydb-library-yql-public-udf-tz-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-public-udf-tz-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-public-udf-tz-ut)
diff --git a/ydb/library/yql/public/udf/tz/ut/CMakeLists.txt b/ydb/library/yql/public/udf/tz/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/public/udf/tz/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/public/udf/tz/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/public/udf/tz/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..fa8472deae7
--- /dev/null
+++ b/ydb/library/yql/public/udf/tz/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,60 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-public-udf-tz-ut)
+target_include_directories(ydb-library-yql-public-udf-tz-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz
+)
+target_link_libraries(ydb-library-yql-public-udf-tz-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ public-udf-tz
+)
+target_sources(ydb-library-yql-public-udf-tz-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/public/udf/tz/udf_tz_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-public-udf-tz-ut
+ TEST_TARGET
+ ydb-library-yql-public-udf-tz-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-public-udf-tz-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-public-udf-tz-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-public-udf-tz-ut)
diff --git a/ydb/library/yql/public/udf/tz/ya.make b/ydb/library/yql/public/udf/tz/ya.make
index 361d9d77915..5cf803181da 100644
--- a/ydb/library/yql/public/udf/tz/ya.make
+++ b/ydb/library/yql/public/udf/tz/ya.make
@@ -12,6 +12,10 @@ PROVIDES(YqlUdfTz)
END()
+RECURSE(
+ gen
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/public/udf/ya.make b/ydb/library/yql/public/udf/ya.make
index 27a6eed2555..a26635cf9ba 100644
--- a/ydb/library/yql/public/udf/ya.make
+++ b/ydb/library/yql/public/udf/ya.make
@@ -53,6 +53,13 @@ PROVIDES(YqlUdfSdk)
END()
+RECURSE(
+ arrow
+ service
+ support
+ tz
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/public/ya.make b/ydb/library/yql/public/ya.make
index bdbac7a8202..676f60c6b3a 100644
--- a/ydb/library/yql/public/ya.make
+++ b/ydb/library/yql/public/ya.make
@@ -4,14 +4,6 @@ RECURSE(
fastcheck
issue
purecalc
- purecalc/examples
- purecalc/helpers
- purecalc/io_specs
types
udf
- udf/arrow
- udf/service/exception_policy
- udf/service/stub
- udf/service/terminate_policy
- udf/tz/gen
)
diff --git a/ydb/library/yql/sql/v0/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v0/CMakeLists.darwin-x86_64.txt
index f2ee61e6f4f..3a805a5ebca 100644
--- a/ydb/library/yql/sql/v0/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/sql/v0/CMakeLists.darwin-x86_64.txt
@@ -7,6 +7,7 @@
add_subdirectory(lexer)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/sql/v0/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v0/CMakeLists.linux-aarch64.txt
index 350fbe3fcc7..52c5ad75fc2 100644
--- a/ydb/library/yql/sql/v0/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/sql/v0/CMakeLists.linux-aarch64.txt
@@ -7,6 +7,7 @@
add_subdirectory(lexer)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/sql/v0/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v0/CMakeLists.linux-x86_64.txt
index 350fbe3fcc7..52c5ad75fc2 100644
--- a/ydb/library/yql/sql/v0/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/sql/v0/CMakeLists.linux-x86_64.txt
@@ -7,6 +7,7 @@
add_subdirectory(lexer)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/sql/v0/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v0/CMakeLists.windows-x86_64.txt
index f2ee61e6f4f..3a805a5ebca 100644
--- a/ydb/library/yql/sql/v0/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/sql/v0/CMakeLists.windows-x86_64.txt
@@ -7,6 +7,7 @@
add_subdirectory(lexer)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/sql/v0/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v0/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..dab39fc9adb
--- /dev/null
+++ b/ydb/library/yql/sql/v0/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,77 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v0-ut)
+target_include_directories(ydb-library-yql-sql-v0-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0
+)
+target_link_libraries(ydb-library-yql-sql-v0-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-sql-v0
+ udf-service-exception_policy
+ library-yql-sql
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-sql-v0-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-sql-v0-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0/sql_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v0-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v0-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ TIMEOUT
+ 300
+)
+target_allocator(ydb-library-yql-sql-v0-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-sql-v0-ut)
diff --git a/ydb/library/yql/sql/v0/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v0/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..69099ea1200
--- /dev/null
+++ b/ydb/library/yql/sql/v0/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,80 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v0-ut)
+target_include_directories(ydb-library-yql-sql-v0-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0
+)
+target_link_libraries(ydb-library-yql-sql-v0-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ yql-sql-v0
+ udf-service-exception_policy
+ library-yql-sql
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-sql-v0-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-sql-v0-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0/sql_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v0-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v0-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ TIMEOUT
+ 300
+)
+target_allocator(ydb-library-yql-sql-v0-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-sql-v0-ut)
diff --git a/ydb/library/yql/sql/v0/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v0/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..d39e61187c3
--- /dev/null
+++ b/ydb/library/yql/sql/v0/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,82 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v0-ut)
+target_include_directories(ydb-library-yql-sql-v0-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0
+)
+target_link_libraries(ydb-library-yql-sql-v0-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-sql-v0
+ udf-service-exception_policy
+ library-yql-sql
+ yql-sql-pg_dummy
+)
+target_link_options(ydb-library-yql-sql-v0-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-sql-v0-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0/sql_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v0-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v0-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ TIMEOUT
+ 300
+)
+target_allocator(ydb-library-yql-sql-v0-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-sql-v0-ut)
diff --git a/ydb/library/yql/sql/v0/ut/CMakeLists.txt b/ydb/library/yql/sql/v0/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/sql/v0/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/sql/v0/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v0/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..e8b26a38f79
--- /dev/null
+++ b/ydb/library/yql/sql/v0/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,70 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v0-ut)
+target_include_directories(ydb-library-yql-sql-v0-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0
+)
+target_link_libraries(ydb-library-yql-sql-v0-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-sql-v0
+ udf-service-exception_policy
+ library-yql-sql
+ yql-sql-pg_dummy
+)
+target_sources(ydb-library-yql-sql-v0-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v0/sql_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v0-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v0-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v0-ut
+ PROPERTY
+ TIMEOUT
+ 300
+)
+target_allocator(ydb-library-yql-sql-v0-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-sql-v0-ut)
diff --git a/ydb/library/yql/sql/v0/ya.make b/ydb/library/yql/sql/v0/ya.make
index 826ad664928..334083a6c90 100644
--- a/ydb/library/yql/sql/v0/ya.make
+++ b/ydb/library/yql/sql/v0/ya.make
@@ -33,6 +33,10 @@ GENERATE_ENUM_SERIALIZATION(node.h)
END()
+RECURSE(
+ lexer
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/sql/v1/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v1/CMakeLists.darwin-x86_64.txt
index 10523a71a47..6e0ae494e7d 100644
--- a/ydb/library/yql/sql/v1/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/sql/v1/CMakeLists.darwin-x86_64.txt
@@ -10,6 +10,7 @@ add_subdirectory(format)
add_subdirectory(lexer)
add_subdirectory(perf)
add_subdirectory(proto_parser)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/sql/v1/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v1/CMakeLists.linux-aarch64.txt
index f01282c6873..17cb39be0ff 100644
--- a/ydb/library/yql/sql/v1/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/sql/v1/CMakeLists.linux-aarch64.txt
@@ -10,6 +10,7 @@ add_subdirectory(format)
add_subdirectory(lexer)
add_subdirectory(perf)
add_subdirectory(proto_parser)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/sql/v1/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v1/CMakeLists.linux-x86_64.txt
index f01282c6873..17cb39be0ff 100644
--- a/ydb/library/yql/sql/v1/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/sql/v1/CMakeLists.linux-x86_64.txt
@@ -10,6 +10,7 @@ add_subdirectory(format)
add_subdirectory(lexer)
add_subdirectory(perf)
add_subdirectory(proto_parser)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/sql/v1/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v1/CMakeLists.windows-x86_64.txt
index 10523a71a47..6e0ae494e7d 100644
--- a/ydb/library/yql/sql/v1/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/sql/v1/CMakeLists.windows-x86_64.txt
@@ -10,6 +10,7 @@ add_subdirectory(format)
add_subdirectory(lexer)
add_subdirectory(perf)
add_subdirectory(proto_parser)
+add_subdirectory(ut)
get_built_tool_path(
TOOL_enum_parser_bin
TOOL_enum_parser_dependency
diff --git a/ydb/library/yql/sql/v1/format/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v1/format/CMakeLists.darwin-x86_64.txt
index 0b8bfe0914a..f5b30fd770a 100644
--- a/ydb/library/yql/sql/v1/format/CMakeLists.darwin-x86_64.txt
+++ b/ydb/library/yql/sql/v1/format/CMakeLists.darwin-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
get_built_tool_path(
TOOL_rescompiler_bin
TOOL_rescompiler_dependency
diff --git a/ydb/library/yql/sql/v1/format/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v1/format/CMakeLists.linux-aarch64.txt
index cb1ff977c68..866b33730a6 100644
--- a/ydb/library/yql/sql/v1/format/CMakeLists.linux-aarch64.txt
+++ b/ydb/library/yql/sql/v1/format/CMakeLists.linux-aarch64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
get_built_tool_path(
TOOL_rescompiler_bin
TOOL_rescompiler_dependency
diff --git a/ydb/library/yql/sql/v1/format/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v1/format/CMakeLists.linux-x86_64.txt
index cb1ff977c68..866b33730a6 100644
--- a/ydb/library/yql/sql/v1/format/CMakeLists.linux-x86_64.txt
+++ b/ydb/library/yql/sql/v1/format/CMakeLists.linux-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
get_built_tool_path(
TOOL_rescompiler_bin
TOOL_rescompiler_dependency
diff --git a/ydb/library/yql/sql/v1/format/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v1/format/CMakeLists.windows-x86_64.txt
index 0b8bfe0914a..f5b30fd770a 100644
--- a/ydb/library/yql/sql/v1/format/CMakeLists.windows-x86_64.txt
+++ b/ydb/library/yql/sql/v1/format/CMakeLists.windows-x86_64.txt
@@ -6,6 +6,7 @@
# original buildsystem will not be accepted.
+add_subdirectory(ut)
get_built_tool_path(
TOOL_rescompiler_bin
TOOL_rescompiler_dependency
diff --git a/ydb/library/yql/sql/v1/format/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v1/format/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..4a924acf871
--- /dev/null
+++ b/ydb/library/yql/sql/v1/format/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,67 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v1-format-ut)
+target_include_directories(ydb-library-yql-sql-v1-format-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format
+)
+target_link_libraries(ydb-library-yql-sql-v1-format-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ sql-v1-format
+)
+target_link_options(ydb-library-yql-sql-v1-format-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-sql-v1-format-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format/sql_format_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v1-format-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v1-format-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-sql-v1-format-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-sql-v1-format-ut)
diff --git a/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..9b78b99e8c0
--- /dev/null
+++ b/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,70 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v1-format-ut)
+target_include_directories(ydb-library-yql-sql-v1-format-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format
+)
+target_link_libraries(ydb-library-yql-sql-v1-format-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ sql-v1-format
+)
+target_link_options(ydb-library-yql-sql-v1-format-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-sql-v1-format-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format/sql_format_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v1-format-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v1-format-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-sql-v1-format-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-sql-v1-format-ut)
diff --git a/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..d8fed6326eb
--- /dev/null
+++ b/ydb/library/yql/sql/v1/format/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,72 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v1-format-ut)
+target_include_directories(ydb-library-yql-sql-v1-format-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format
+)
+target_link_libraries(ydb-library-yql-sql-v1-format-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ sql-v1-format
+)
+target_link_options(ydb-library-yql-sql-v1-format-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-sql-v1-format-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format/sql_format_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v1-format-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v1-format-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-sql-v1-format-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-sql-v1-format-ut)
diff --git a/ydb/library/yql/sql/v1/format/ut/CMakeLists.txt b/ydb/library/yql/sql/v1/format/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/sql/v1/format/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/sql/v1/format/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v1/format/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..c130d878505
--- /dev/null
+++ b/ydb/library/yql/sql/v1/format/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,60 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v1-format-ut)
+target_include_directories(ydb-library-yql-sql-v1-format-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format
+)
+target_link_libraries(ydb-library-yql-sql-v1-format-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ sql-v1-format
+)
+target_sources(ydb-library-yql-sql-v1-format-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/format/sql_format_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v1-format-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v1-format-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ LABELS
+ SMALL
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-format-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+target_allocator(ydb-library-yql-sql-v1-format-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-sql-v1-format-ut)
diff --git a/ydb/library/yql/sql/v1/ut/CMakeLists.darwin-x86_64.txt b/ydb/library/yql/sql/v1/ut/CMakeLists.darwin-x86_64.txt
new file mode 100644
index 00000000000..17379347b27
--- /dev/null
+++ b/ydb/library/yql/sql/v1/ut/CMakeLists.darwin-x86_64.txt
@@ -0,0 +1,79 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v1-ut)
+target_include_directories(ydb-library-yql-sql-v1-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1
+)
+target_link_libraries(ydb-library-yql-sql-v1-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-sql-v1
+ udf-service-exception_policy
+ library-yql-sql
+ yql-sql-pg_dummy
+ sql-v1-format
+)
+target_link_options(ydb-library-yql-sql-v1-ut PRIVATE
+ -Wl,-platform_version,macos,11.0,11.0
+ -fPIC
+ -fPIC
+ -framework
+ CoreFoundation
+)
+target_sources(ydb-library-yql-sql-v1-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_match_recognize_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v1-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v1-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ TIMEOUT
+ 300
+)
+target_allocator(ydb-library-yql-sql-v1-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-sql-v1-ut)
diff --git a/ydb/library/yql/sql/v1/ut/CMakeLists.linux-aarch64.txt b/ydb/library/yql/sql/v1/ut/CMakeLists.linux-aarch64.txt
new file mode 100644
index 00000000000..e752dcf4475
--- /dev/null
+++ b/ydb/library/yql/sql/v1/ut/CMakeLists.linux-aarch64.txt
@@ -0,0 +1,82 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v1-ut)
+target_include_directories(ydb-library-yql-sql-v1-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1
+)
+target_link_libraries(ydb-library-yql-sql-v1-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ cpp-testing-unittest_main
+ yql-sql-v1
+ udf-service-exception_policy
+ library-yql-sql
+ yql-sql-pg_dummy
+ sql-v1-format
+)
+target_link_options(ydb-library-yql-sql-v1-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-sql-v1-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_match_recognize_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v1-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v1-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ TIMEOUT
+ 300
+)
+target_allocator(ydb-library-yql-sql-v1-ut
+ cpp-malloc-jemalloc
+)
+vcs_info(ydb-library-yql-sql-v1-ut)
diff --git a/ydb/library/yql/sql/v1/ut/CMakeLists.linux-x86_64.txt b/ydb/library/yql/sql/v1/ut/CMakeLists.linux-x86_64.txt
new file mode 100644
index 00000000000..cbe9969c009
--- /dev/null
+++ b/ydb/library/yql/sql/v1/ut/CMakeLists.linux-x86_64.txt
@@ -0,0 +1,84 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v1-ut)
+target_include_directories(ydb-library-yql-sql-v1-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1
+)
+target_link_libraries(ydb-library-yql-sql-v1-ut PUBLIC
+ contrib-libs-linux-headers
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-sql-v1
+ udf-service-exception_policy
+ library-yql-sql
+ yql-sql-pg_dummy
+ sql-v1-format
+)
+target_link_options(ydb-library-yql-sql-v1-ut PRIVATE
+ -ldl
+ -lrt
+ -Wl,--no-as-needed
+ -fPIC
+ -fPIC
+ -lpthread
+ -lrt
+ -ldl
+)
+target_sources(ydb-library-yql-sql-v1-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_match_recognize_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v1-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v1-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ TIMEOUT
+ 300
+)
+target_allocator(ydb-library-yql-sql-v1-ut
+ cpp-malloc-tcmalloc
+ libs-tcmalloc-no_percpu_cache
+)
+vcs_info(ydb-library-yql-sql-v1-ut)
diff --git a/ydb/library/yql/sql/v1/ut/CMakeLists.txt b/ydb/library/yql/sql/v1/ut/CMakeLists.txt
new file mode 100644
index 00000000000..f8b31df0c11
--- /dev/null
+++ b/ydb/library/yql/sql/v1/ut/CMakeLists.txt
@@ -0,0 +1,17 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+if (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "aarch64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-aarch64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Darwin" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64")
+ include(CMakeLists.darwin-x86_64.txt)
+elseif (WIN32 AND CMAKE_SYSTEM_PROCESSOR STREQUAL "AMD64" AND NOT HAVE_CUDA)
+ include(CMakeLists.windows-x86_64.txt)
+elseif (CMAKE_SYSTEM_NAME STREQUAL "Linux" AND CMAKE_SYSTEM_PROCESSOR STREQUAL "x86_64" AND NOT HAVE_CUDA)
+ include(CMakeLists.linux-x86_64.txt)
+endif()
diff --git a/ydb/library/yql/sql/v1/ut/CMakeLists.windows-x86_64.txt b/ydb/library/yql/sql/v1/ut/CMakeLists.windows-x86_64.txt
new file mode 100644
index 00000000000..2352e1c720f
--- /dev/null
+++ b/ydb/library/yql/sql/v1/ut/CMakeLists.windows-x86_64.txt
@@ -0,0 +1,72 @@
+
+# This file was generated by the build system used internally in the Yandex monorepo.
+# Only simple modifications are allowed (adding source-files to targets, adding simple properties
+# like target_include_directories). These modifications will be ported to original
+# ya.make files by maintainers. Any complex modifications which can't be ported back to the
+# original buildsystem will not be accepted.
+
+
+
+add_executable(ydb-library-yql-sql-v1-ut)
+target_include_directories(ydb-library-yql-sql-v1-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1
+)
+target_link_libraries(ydb-library-yql-sql-v1-ut PUBLIC
+ contrib-libs-cxxsupp
+ yutil
+ library-cpp-cpuid_check
+ cpp-testing-unittest_main
+ yql-sql-v1
+ udf-service-exception_policy
+ library-yql-sql
+ yql-sql-pg_dummy
+ sql-v1-format
+)
+target_sources(ydb-library-yql-sql-v1-ut PRIVATE
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_ut.cpp
+ ${CMAKE_SOURCE_DIR}/ydb/library/yql/sql/v1/sql_match_recognize_ut.cpp
+)
+set_property(
+ TARGET
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ SPLIT_FACTOR
+ 1
+)
+add_yunittest(
+ NAME
+ ydb-library-yql-sql-v1-ut
+ TEST_TARGET
+ ydb-library-yql-sql-v1-ut
+ TEST_ARG
+ --print-before-suite
+ --print-before-test
+ --fork-tests
+ --print-times
+ --show-fails
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ LABELS
+ MEDIUM
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ PROCESSORS
+ 1
+)
+set_yunittest_property(
+ TEST
+ ydb-library-yql-sql-v1-ut
+ PROPERTY
+ TIMEOUT
+ 300
+)
+target_allocator(ydb-library-yql-sql-v1-ut
+ system_allocator
+)
+vcs_info(ydb-library-yql-sql-v1-ut)
diff --git a/ydb/library/yql/sql/v1/ya.make b/ydb/library/yql/sql/v1/ya.make
index 5ecab38ae67..c6d812c707c 100644
--- a/ydb/library/yql/sql/v1/ya.make
+++ b/ydb/library/yql/sql/v1/ya.make
@@ -56,6 +56,13 @@ GENERATE_ENUM_SERIALIZATION(sql_call_param.h)
END()
+RECURSE(
+ format
+ lexer
+ perf
+ proto_parser
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/sql/ya.make b/ydb/library/yql/sql/ya.make
index 9d4ff1450c5..92837d28714 100644
--- a/ydb/library/yql/sql/ya.make
+++ b/ydb/library/yql/sql/ya.make
@@ -23,4 +23,7 @@ END()
RECURSE(
pg
pg_dummy
+ settings
+ v0
+ v1
)
diff --git a/ydb/library/yql/tools/mrjob/ya.make b/ydb/library/yql/tools/mrjob/ya.make
index c91ff06cda8..95ae41ec65c 100644
--- a/ydb/library/yql/tools/mrjob/ya.make
+++ b/ydb/library/yql/tools/mrjob/ya.make
@@ -25,3 +25,7 @@ PEERDIR(
YQL_LAST_ABI_VERSION()
END()
+
+RECURSE_FOR_TESTS(
+ test
+)
diff --git a/ydb/library/yql/tools/ya.make b/ydb/library/yql/tools/ya.make
index 062df88349a..3e928cff45d 100644
--- a/ydb/library/yql/tools/ya.make
+++ b/ydb/library/yql/tools/ya.make
@@ -1,6 +1,5 @@
RECURSE(
astdiff
mrjob
- mrjob/test
yqlrun
)
diff --git a/ydb/library/yql/utils/backtrace/ya.make b/ydb/library/yql/utils/backtrace/ya.make
index 93d53065807..15836994993 100644
--- a/ydb/library/yql/utils/backtrace/ya.make
+++ b/ydb/library/yql/utils/backtrace/ya.make
@@ -29,6 +29,10 @@ PEERDIR(
END()
+RECURSE(
+ fake_llvm_symbolizer
+)
+
RECURSE_FOR_TESTS(
ut
)
diff --git a/ydb/library/yql/utils/ya.make b/ydb/library/yql/utils/ya.make
index f5ea18eb6c5..e8df4b5e156 100644
--- a/ydb/library/yql/utils/ya.make
+++ b/ydb/library/yql/utils/ya.make
@@ -57,6 +57,7 @@ RECURSE(
failure_injector
fetch
log
+ test_http_server
threading
)
diff --git a/ydb/library/yql/ya.make b/ydb/library/yql/ya.make
index 83b1b9e9270..13581288157 100644
--- a/ydb/library/yql/ya.make
+++ b/ydb/library/yql/ya.make
@@ -1,21 +1,6 @@
RECURSE(
ast
core
- core/arrow_kernels/request
- core/arrow_kernels/registry
- core/common_opt
- core/credentials
- core/expr_nodes
- core/expr_nodes_gen
- core/extract_predicate
- core/facade
- core/file_storage
- core/issue
- core/peephole_opt
- core/services
- core/sql_types
- core/type_ann
- core/user_data
dq
minikql
parser