aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorudovichenko-r <udovichenko-r@yandex-team.ru>2022-02-10 16:49:21 +0300
committerDaniil Cherednik <dcherednik@yandex-team.ru>2022-02-10 16:49:21 +0300
commitd7e4eaec9d325e188dabb3eb1949a32a5229e9ce (patch)
tree75f143caa4e1e35b760846b22310ede3212d7f63
parent027f7b7f6644e258a5840869b9b82c79e1ebeee0 (diff)
downloadydb-d7e4eaec9d325e188dabb3eb1949a32a5229e9ce.tar.gz
Restoring authorship annotation for <udovichenko-r@yandex-team.ru>. Commit 1 of 2.
-rw-r--r--build/conf/project_specific/yql_udf.conf2
-rw-r--r--library/cpp/binsaver/bin_saver.h52
-rw-r--r--library/cpp/codecs/codecs_registry.cpp4
-rw-r--r--library/cpp/codecs/solar_codec.h54
-rw-r--r--library/cpp/codecs/ut/codecs_ut.cpp12
-rw-r--r--library/cpp/containers/comptrie/comptrie_impl.h60
-rw-r--r--library/cpp/containers/comptrie/comptrie_trie.h46
-rw-r--r--library/cpp/containers/comptrie/comptrie_ut.cpp116
-rw-r--r--library/cpp/containers/comptrie/protopacker.h2
-rw-r--r--library/cpp/containers/sorted_vector/ut/ya.make2
-rw-r--r--library/cpp/containers/ya.make4
-rw-r--r--library/cpp/ya.make10
-rw-r--r--library/cpp/yson/node/node.cpp6
-rw-r--r--library/cpp/yson/writer.cpp2
-rw-r--r--library/cpp/yson/writer.h2
-rw-r--r--tools/ya.make2
-rw-r--r--util/generic/algorithm.h8
-rw-r--r--util/generic/bitmap.h1222
-rw-r--r--util/generic/bitmap_ut.cpp772
-rw-r--r--util/generic/ptr.h2
-rw-r--r--util/stream/debug.cpp12
-rw-r--r--util/stream/debug.h2
-rw-r--r--util/stream/trace.h28
-rw-r--r--ydb/core/client/minikql_compile/yql_expr_minikql.cpp66
-rw-r--r--ydb/core/engine/minikql/flat_local_tx_minikql.h2
-rw-r--r--ydb/core/engine/mkql_engine_flat.cpp4
-rw-r--r--ydb/core/kqp/host/kqp_host.cpp28
-rw-r--r--ydb/core/kqp/host/kqp_run_prepared.cpp2
-rw-r--r--ydb/core/kqp/host/kqp_runner.cpp6
-rw-r--r--ydb/core/kqp/prepare/kqp_query_analyze.cpp20
-rw-r--r--ydb/core/kqp/prepare/kqp_query_exec.cpp10
-rw-r--r--ydb/core/kqp/prepare/kqp_query_finalize.cpp36
-rw-r--r--ydb/core/kqp/prepare/kqp_query_rewrite.cpp64
-rw-r--r--ydb/core/kqp/prepare/kqp_query_simplify.cpp24
-rw-r--r--ydb/core/kqp/prepare/kqp_query_substitute.cpp8
-rw-r--r--ydb/core/kqp/prepare/kqp_type_ann.cpp2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_datasink.cpp12
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_datasource.cpp38
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_exec.cpp14
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_expr_nodes.cpp4
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_expr_nodes.h4
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_expr_nodes.json68
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_mkql.cpp2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt.cpp36
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt_build.cpp2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt_join.cpp2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_opt_range.cpp2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_provider.cpp2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_provider_impl.h4
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_results.cpp2
-rw-r--r--ydb/core/kqp/provider/yql_kikimr_type_ann.cpp100
-rw-r--r--ydb/core/kqp/ut/kqp_join_ut.cpp4
-rw-r--r--ydb/core/scheme_types/scheme_type_traits.h78
-rw-r--r--ydb/core/testlib/minikql_compile.h2
-rw-r--r--ydb/core/util/tracking_alloc.h136
-rw-r--r--ydb/core/yq/libs/gateway/empty_gateway.cpp6
-rw-r--r--ydb/core/yql_testlib/yql_testlib.cpp2
-rw-r--r--ydb/library/yql/ast/ya.make6
-rw-r--r--ydb/library/yql/ast/yql_ast.h2
-rw-r--r--ydb/library/yql/ast/yql_ast_ut.cpp38
-rw-r--r--ydb/library/yql/ast/yql_constraint.cpp1556
-rw-r--r--ydb/library/yql/ast/yql_constraint.h624
-rw-r--r--ydb/library/yql/ast/yql_expr.cpp236
-rw-r--r--ydb/library/yql/ast/yql_expr.h276
-rw-r--r--ydb/library/yql/ast/yql_expr_builder.cpp34
-rw-r--r--ydb/library/yql/ast/yql_expr_builder.h8
-rw-r--r--ydb/library/yql/ast/yql_expr_ut.cpp12
-rw-r--r--ydb/library/yql/core/common_opt/ya.make10
-rw-r--r--ydb/library/yql/core/common_opt/yql_co.h46
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_extr_members.cpp1074
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_extr_members.h48
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_finalizers.cpp370
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_flow1.cpp998
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_flow2.cpp808
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_simple1.cpp1474
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_simple2.cpp42
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_transformer.cpp94
-rw-r--r--ydb/library/yql/core/common_opt/yql_co_transformer.h4
-rw-r--r--ydb/library/yql/core/expr_nodes/ya.make6
-rw-r--r--ydb/library/yql/core/expr_nodes/yql_expr_nodes.h158
-rw-r--r--ydb/library/yql/core/expr_nodes/yql_expr_nodes.json1750
-rw-r--r--ydb/library/yql/core/expr_nodes_gen/gen/__main__.py10
-rw-r--r--ydb/library/yql/core/expr_nodes_gen/ya.make6
-rw-r--r--ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h154
-rw-r--r--ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj116
-rw-r--r--ydb/library/yql/core/extract_predicate/extract_predicate_impl.cpp4
-rw-r--r--ydb/library/yql/core/facade/yql_facade.cpp434
-rw-r--r--ydb/library/yql/core/facade/yql_facade.h78
-rw-r--r--ydb/library/yql/core/file_storage/file_storage.cpp264
-rw-r--r--ydb/library/yql/core/file_storage/file_storage.h66
-rw-r--r--ydb/library/yql/core/file_storage/file_storage_ut.cpp104
-rw-r--r--ydb/library/yql/core/file_storage/http_download/http_download.cpp344
-rw-r--r--ydb/library/yql/core/file_storage/http_download/http_download.h26
-rw-r--r--ydb/library/yql/core/file_storage/http_download/pattern_group.cpp2
-rw-r--r--ydb/library/yql/core/file_storage/http_download/pattern_group_ut.cpp2
-rw-r--r--ydb/library/yql/core/file_storage/http_download/ut/ya.make18
-rw-r--r--ydb/library/yql/core/file_storage/http_download/ya.make50
-rw-r--r--ydb/library/yql/core/file_storage/proto/file_storage.proto24
-rw-r--r--ydb/library/yql/core/file_storage/proto/ya.make16
-rw-r--r--ydb/library/yql/core/file_storage/sized_cache.cpp174
-rw-r--r--ydb/library/yql/core/file_storage/sized_cache.h134
-rw-r--r--ydb/library/yql/core/file_storage/sized_cache_ut.cpp290
-rw-r--r--ydb/library/yql/core/file_storage/storage.cpp348
-rw-r--r--ydb/library/yql/core/file_storage/storage.h72
-rw-r--r--ydb/library/yql/core/file_storage/storage_ut.cpp166
-rw-r--r--ydb/library/yql/core/file_storage/url_mapper_ut.cpp20
-rw-r--r--ydb/library/yql/core/file_storage/ut/ya.make18
-rw-r--r--ydb/library/yql/core/file_storage/ya.make30
-rw-r--r--ydb/library/yql/core/issue/protos/issue_id.proto30
-rw-r--r--ydb/library/yql/core/issue/yql_issue.txt138
-rw-r--r--ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp304
-rw-r--r--ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h4
-rw-r--r--ydb/library/yql/core/services/mounts/yql_mounts.cpp6
-rw-r--r--ydb/library/yql/core/services/mounts/yql_mounts.h4
-rw-r--r--ydb/library/yql/core/services/yql_eval_expr.cpp282
-rw-r--r--ydb/library/yql/core/services/yql_eval_params.cpp6
-rw-r--r--ydb/library/yql/core/services/yql_out_transformers.cpp34
-rw-r--r--ydb/library/yql/core/services/yql_out_transformers.h60
-rw-r--r--ydb/library/yql/core/services/yql_plan.cpp54
-rw-r--r--ydb/library/yql/core/services/yql_transform_pipeline.cpp114
-rw-r--r--ydb/library/yql/core/services/yql_transform_pipeline.h42
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_core.cpp2354
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_expr.cpp66
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_expr.h2
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_impl.h4
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_join.cpp164
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_list.cpp748
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_list.h4
-rw-r--r--ydb/library/yql/core/type_ann/type_ann_types.cpp180
-rw-r--r--ydb/library/yql/core/type_ann/ya.make6
-rw-r--r--ydb/library/yql/core/ya.make14
-rw-r--r--ydb/library/yql/core/yql_callable_names.h16
-rw-r--r--ydb/library/yql/core/yql_callable_transform.cpp2
-rw-r--r--ydb/library/yql/core/yql_callable_transform.h532
-rw-r--r--ydb/library/yql/core/yql_csv.cpp10
-rw-r--r--ydb/library/yql/core/yql_csv.h2
-rw-r--r--ydb/library/yql/core/yql_data_provider.h50
-rw-r--r--ydb/library/yql/core/yql_execution.cpp276
-rw-r--r--ydb/library/yql/core/yql_execution.h22
-rw-r--r--ydb/library/yql/core/yql_expr_constraint.cpp4326
-rw-r--r--ydb/library/yql/core/yql_expr_constraint.h32
-rw-r--r--ydb/library/yql/core/yql_expr_csee.cpp28
-rw-r--r--ydb/library/yql/core/yql_expr_optimize.cpp242
-rw-r--r--ydb/library/yql/core/yql_expr_optimize.h20
-rw-r--r--ydb/library/yql/core/yql_expr_type_annotation.cpp464
-rw-r--r--ydb/library/yql/core/yql_expr_type_annotation.h8
-rw-r--r--ydb/library/yql/core/yql_graph_transformer.h68
-rw-r--r--ydb/library/yql/core/yql_join.cpp48
-rw-r--r--ydb/library/yql/core/yql_join.h18
-rw-r--r--ydb/library/yql/core/yql_opt_proposed_by_data.cpp332
-rw-r--r--ydb/library/yql/core/yql_opt_proposed_by_data.h10
-rw-r--r--ydb/library/yql/core/yql_opt_rewrite_io.cpp10
-rw-r--r--ydb/library/yql/core/yql_opt_utils.cpp648
-rw-r--r--ydb/library/yql/core/yql_opt_utils.h52
-rw-r--r--ydb/library/yql/core/yql_type_annotation.cpp38
-rw-r--r--ydb/library/yql/core/yql_type_annotation.h124
-rw-r--r--ydb/library/yql/core/yql_type_helpers.cpp40
-rw-r--r--ydb/library/yql/core/yql_type_helpers.h4
-rw-r--r--ydb/library/yql/dq/opt/dq_opt.cpp8
-rw-r--r--ydb/library/yql/dq/opt/dq_opt.h2
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_join.cpp2
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_log.cpp32
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_log.h14
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_phy.cpp58
-rw-r--r--ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp12
-rw-r--r--ydb/library/yql/dq/runtime/dq_tasks_runner.cpp8
-rw-r--r--ydb/library/yql/dq/runtime/dq_tasks_runner.h2
-rw-r--r--ydb/library/yql/dq/type_ann/dq_type_ann.cpp696
-rw-r--r--ydb/library/yql/dq/type_ann/dq_type_ann.h18
-rw-r--r--ydb/library/yql/minikql/aligned_page_pool.cpp176
-rw-r--r--ydb/library/yql/minikql/aligned_page_pool.h102
-rw-r--r--ydb/library/yql/minikql/benchmark/pack_num/metrics/main.py12
-rw-r--r--ydb/library/yql/minikql/benchmark/pack_num/metrics/ya.make36
-rw-r--r--ydb/library/yql/minikql/benchmark/pack_num/pack_num_bench.cpp446
-rw-r--r--ydb/library/yql/minikql/benchmark/pack_num/ya.make32
-rw-r--r--ydb/library/yql/minikql/benchmark/ya.make16
-rw-r--r--ydb/library/yql/minikql/codegen/codegen.cpp66
-rw-r--r--ydb/library/yql/minikql/codegen/codegen.h2
-rw-r--r--ydb/library/yql/minikql/codegen/codegen_ut.cpp16
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp18
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp74
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_combine.h20
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_factories.h4
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp30
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp6
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_group.cpp384
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_group.h20
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp2
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_join.cpp18
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp4
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp12
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp112
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp94
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp90
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_varitem.h18
-rw-r--r--ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp4
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp224
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp340
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp2
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp394
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp214
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ut/ya.make6
-rw-r--r--ydb/library/yql/minikql/comp_nodes/ya.make12
-rw-r--r--ydb/library/yql/minikql/compact_hash.cpp120
-rw-r--r--ydb/library/yql/minikql/compact_hash.h3094
-rw-r--r--ydb/library/yql/minikql/compact_hash_ut.cpp740
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_list_adapter.h2
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node.cpp80
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node.h92
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp8
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h28
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp232
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp18
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp484
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_holders.h112
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_impl.h4
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp746
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_pack.h78
-rw-r--r--ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp428
-rw-r--r--ydb/library/yql/minikql/computation/mkql_validate_ut.cpp2
-rw-r--r--ydb/library/yql/minikql/computation/ut/ya.make4
-rw-r--r--ydb/library/yql/minikql/dom/convert.h4
-rw-r--r--ydb/library/yql/minikql/dom/peel.cpp2
-rw-r--r--ydb/library/yql/minikql/mkql_alloc.cpp18
-rw-r--r--ydb/library/yql/minikql/mkql_alloc.h44
-rw-r--r--ydb/library/yql/minikql/mkql_alloc_ut.cpp2
-rw-r--r--ydb/library/yql/minikql/mkql_function_registry.cpp10
-rw-r--r--ydb/library/yql/minikql/mkql_mem_info.h12
-rw-r--r--ydb/library/yql/minikql/mkql_node.cpp2
-rw-r--r--ydb/library/yql/minikql/mkql_node.h4
-rw-r--r--ydb/library/yql/minikql/mkql_node_serialization.cpp28
-rw-r--r--ydb/library/yql/minikql/mkql_program_builder.cpp122
-rw-r--r--ydb/library/yql/minikql/mkql_program_builder.h12
-rw-r--r--ydb/library/yql/minikql/mkql_type_builder.cpp26
-rw-r--r--ydb/library/yql/minikql/mkql_type_builder.h24
-rw-r--r--ydb/library/yql/minikql/mkql_type_ops.cpp4
-rw-r--r--ydb/library/yql/minikql/pack_num.cpp2
-rw-r--r--ydb/library/yql/minikql/pack_num.h324
-rw-r--r--ydb/library/yql/minikql/pack_num_ut.cpp128
-rw-r--r--ydb/library/yql/minikql/perf/packer/packer.cpp100
-rw-r--r--ydb/library/yql/minikql/perf/packer/ya.make34
-rw-r--r--ydb/library/yql/minikql/perf/presort/presort.cpp2
-rw-r--r--ydb/library/yql/minikql/primes.cpp40
-rw-r--r--ydb/library/yql/minikql/primes.gen102
-rw-r--r--ydb/library/yql/minikql/primes.h14
-rw-r--r--ydb/library/yql/minikql/ya.make2
-rw-r--r--ydb/library/yql/mount/lib/yql/core.yql44
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/ya.make4
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp14
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp58
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.h22
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp2
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_logical_opt.cpp36
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_mkql_compiler.cpp126
-rw-r--r--ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_mkql_compiler.h18
-rw-r--r--ydb/library/yql/providers/common/codec/ya.make8
-rw-r--r--ydb/library/yql/providers/common/codec/yql_codec.cpp362
-rw-r--r--ydb/library/yql/providers/common/codec/yql_codec.h38
-rw-r--r--ydb/library/yql/providers/common/codec/yql_codec_buf.cpp52
-rw-r--r--ydb/library/yql/providers/common/codec/yql_codec_buf.h66
-rw-r--r--ydb/library/yql/providers/common/codec/yql_codec_results.cpp2
-rw-r--r--ydb/library/yql/providers/common/codec/yql_codec_results.h102
-rw-r--r--ydb/library/yql/providers/common/codec/yql_codec_type_flags.cpp2
-rw-r--r--ydb/library/yql/providers/common/codec/yql_codec_type_flags.h56
-rw-r--r--ydb/library/yql/providers/common/comp_nodes/ya.make2
-rw-r--r--ydb/library/yql/providers/common/comp_nodes/yql_factory.cpp24
-rw-r--r--ydb/library/yql/providers/common/comp_nodes/yql_factory.h2
-rw-r--r--ydb/library/yql/providers/common/comp_nodes/yql_maketype.cpp6
-rw-r--r--ydb/library/yql/providers/common/comp_nodes/yql_reprcode.cpp4
-rw-r--r--ydb/library/yql/providers/common/comp_nodes/yql_serializetypehandle.cpp2
-rw-r--r--ydb/library/yql/providers/common/comp_nodes/yql_typehandle.cpp4
-rw-r--r--ydb/library/yql/providers/common/config/ya.make28
-rw-r--r--ydb/library/yql/providers/common/config/yql_configuration_transformer.cpp144
-rw-r--r--ydb/library/yql/providers/common/config/yql_configuration_transformer.h46
-rw-r--r--ydb/library/yql/providers/common/config/yql_dispatch.cpp340
-rw-r--r--ydb/library/yql/providers/common/config/yql_dispatch.h678
-rw-r--r--ydb/library/yql/providers/common/config/yql_setting.h256
-rw-r--r--ydb/library/yql/providers/common/dq/ya.make28
-rw-r--r--ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp98
-rw-r--r--ydb/library/yql/providers/common/dq/yql_dq_integration_impl.h36
-rw-r--r--ydb/library/yql/providers/common/gateway/yql_provider_gateway.cpp30
-rw-r--r--ydb/library/yql/providers/common/gateway/yql_provider_gateway.h8
-rw-r--r--ydb/library/yql/providers/common/metrics/metrics_registry.cpp2
-rw-r--r--ydb/library/yql/providers/common/metrics/metrics_registry.h8
-rw-r--r--ydb/library/yql/providers/common/mkql/ya.make6
-rw-r--r--ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp132
-rw-r--r--ydb/library/yql/providers/common/mkql/yql_provider_mkql.h2
-rw-r--r--ydb/library/yql/providers/common/mkql/yql_type_mkql.cpp38
-rw-r--r--ydb/library/yql/providers/common/mkql/yql_type_mkql.h2
-rw-r--r--ydb/library/yql/providers/common/proto/gateways_config.proto150
-rw-r--r--ydb/library/yql/providers/common/proto/udf_resolver.proto72
-rw-r--r--ydb/library/yql/providers/common/proto/ya.make2
-rw-r--r--ydb/library/yql/providers/common/provider/ya.make4
-rw-r--r--ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp420
-rw-r--r--ydb/library/yql/providers/common/provider/yql_data_provider_impl.h122
-rw-r--r--ydb/library/yql/providers/common/provider/yql_provider.cpp490
-rw-r--r--ydb/library/yql/providers/common/provider/yql_provider.h42
-rw-r--r--ydb/library/yql/providers/common/provider/yql_table_lookup.cpp56
-rw-r--r--ydb/library/yql/providers/common/schema/expr/ya.make22
-rw-r--r--ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp212
-rw-r--r--ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h28
-rw-r--r--ydb/library/yql/providers/common/schema/mkql/ya.make26
-rw-r--r--ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.cpp286
-rw-r--r--ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h18
-rw-r--r--ydb/library/yql/providers/common/schema/parser/ya.make28
-rw-r--r--ydb/library/yql/providers/common/schema/parser/yql_type_parser.cpp74
-rw-r--r--ydb/library/yql/providers/common/schema/parser/yql_type_parser.h818
-rw-r--r--ydb/library/yql/providers/common/schema/skiff/ya.make22
-rw-r--r--ydb/library/yql/providers/common/schema/skiff/yql_skiff_schema.cpp420
-rw-r--r--ydb/library/yql/providers/common/schema/skiff/yql_skiff_schema.h4
-rw-r--r--ydb/library/yql/providers/common/schema/ya.make26
-rw-r--r--ydb/library/yql/providers/common/schema/yql_schema_utils.cpp206
-rw-r--r--ydb/library/yql/providers/common/schema/yql_schema_utils.h10
-rw-r--r--ydb/library/yql/providers/common/transform/ya.make28
-rw-r--r--ydb/library/yql/providers/common/transform/yql_exec.cpp188
-rw-r--r--ydb/library/yql/providers/common/transform/yql_exec.h120
-rw-r--r--ydb/library/yql/providers/common/transform/yql_lazy_init.cpp2
-rw-r--r--ydb/library/yql/providers/common/transform/yql_lazy_init.h60
-rw-r--r--ydb/library/yql/providers/common/transform/yql_optimize.cpp408
-rw-r--r--ydb/library/yql/providers/common/transform/yql_optimize.h206
-rw-r--r--ydb/library/yql/providers/common/transform/yql_visit.cpp54
-rw-r--r--ydb/library/yql/providers/common/transform/yql_visit.h120
-rw-r--r--ydb/library/yql/providers/common/udf_resolve/ya.make36
-rw-r--r--ydb/library/yql/providers/common/udf_resolve/yql_files_box.cpp64
-rw-r--r--ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp250
-rw-r--r--ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.h22
-rw-r--r--ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp70
-rw-r--r--ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h4
-rw-r--r--ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp4
-rw-r--r--ydb/library/yql/providers/common/ya.make20
-rw-r--r--ydb/library/yql/providers/config/yql_config_provider.cpp1072
-rw-r--r--ydb/library/yql/providers/config/yql_config_provider.h6
-rw-r--r--ydb/library/yql/providers/dq/actors/actor_helpers.h2
-rw-r--r--ydb/library/yql/providers/dq/actors/executer_actor.cpp18
-rw-r--r--ydb/library/yql/providers/dq/actors/full_result_writer.cpp100
-rw-r--r--ydb/library/yql/providers/dq/actors/full_result_writer.h8
-rw-r--r--ydb/library/yql/providers/dq/actors/graph_execution_events_actor.cpp160
-rw-r--r--ydb/library/yql/providers/dq/actors/graph_execution_events_actor.h6
-rw-r--r--ydb/library/yql/providers/dq/actors/proto_builder.cpp286
-rw-r--r--ydb/library/yql/providers/dq/actors/proto_builder.h44
-rw-r--r--ydb/library/yql/providers/dq/actors/result_aggregator.cpp998
-rw-r--r--ydb/library/yql/providers/dq/actors/result_aggregator.h4
-rw-r--r--ydb/library/yql/providers/dq/actors/result_receiver.cpp2
-rw-r--r--ydb/library/yql/providers/dq/actors/task_controller.cpp4
-rw-r--r--ydb/library/yql/providers/dq/actors/worker_actor.cpp4
-rw-r--r--ydb/library/yql/providers/dq/actors/ya.make12
-rw-r--r--ydb/library/yql/providers/dq/api/protos/dqs.proto14
-rw-r--r--ydb/library/yql/providers/dq/api/protos/service.proto6
-rw-r--r--ydb/library/yql/providers/dq/common/yql_dq_settings.cpp6
-rw-r--r--ydb/library/yql/providers/dq/common/yql_dq_settings.h18
-rw-r--r--ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.json28
-rw-r--r--ydb/library/yql/providers/dq/interface/ya.make28
-rw-r--r--ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.cpp2
-rw-r--r--ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.h30
-rw-r--r--ydb/library/yql/providers/dq/interface/yql_dq_integration.cpp2
-rw-r--r--ydb/library/yql/providers/dq/interface/yql_dq_integration.h66
-rw-r--r--ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.cpp2
-rw-r--r--ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.h18
-rw-r--r--ydb/library/yql/providers/dq/interface/yql_dq_task_transform.cpp42
-rw-r--r--ydb/library/yql/providers/dq/interface/yql_dq_task_transform.h30
-rw-r--r--ydb/library/yql/providers/dq/local_gateway/ya.make4
-rw-r--r--ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp34
-rw-r--r--ydb/library/yql/providers/dq/mkql/dqs_mkql_compiler.cpp32
-rw-r--r--ydb/library/yql/providers/dq/mkql/dqs_mkql_compiler.h24
-rw-r--r--ydb/library/yql/providers/dq/mkql/ya.make28
-rw-r--r--ydb/library/yql/providers/dq/opt/dqs_opt.cpp38
-rw-r--r--ydb/library/yql/providers/dq/opt/dqs_opt.h2
-rw-r--r--ydb/library/yql/providers/dq/opt/logical_optimize.cpp146
-rw-r--r--ydb/library/yql/providers/dq/opt/logical_optimize.h18
-rw-r--r--ydb/library/yql/providers/dq/opt/physical_optimize.cpp240
-rw-r--r--ydb/library/yql/providers/dq/opt/physical_optimize.h20
-rw-r--r--ydb/library/yql/providers/dq/opt/ya.make4
-rw-r--r--ydb/library/yql/providers/dq/planner/dqs_task_graph.h6
-rw-r--r--ydb/library/yql/providers/dq/planner/execution_planner.cpp62
-rw-r--r--ydb/library/yql/providers/dq/planner/execution_planner.h16
-rw-r--r--ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp354
-rw-r--r--ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h6
-rw-r--r--ydb/library/yql/providers/dq/provider/ya.make14
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_control.h8
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp96
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_datasink.h2
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.cpp62
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.h18
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_datasource.cpp120
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_datasource.h6
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_datasource_type_ann.cpp132
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_datasource_type_ann.h26
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp52
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_gateway.h20
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp6
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_provider.h2
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp272
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_recapture.h24
-rw-r--r--ydb/library/yql/providers/dq/provider/yql_dq_state.h14
-rw-r--r--ydb/library/yql/providers/dq/runtime/task_command_executor.cpp56
-rw-r--r--ydb/library/yql/providers/dq/runtime/task_command_executor.h18
-rw-r--r--ydb/library/yql/providers/dq/service/grpc_service.cpp122
-rw-r--r--ydb/library/yql/providers/dq/task_runner/file_cache.cpp2
-rw-r--r--ydb/library/yql/providers/dq/task_runner/file_cache.h4
-rw-r--r--ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp48
-rw-r--r--ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h8
-rw-r--r--ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp22
-rw-r--r--ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.h10
-rw-r--r--ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.cpp8
-rw-r--r--ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h4
-rw-r--r--ydb/library/yql/providers/dq/task_runner/ya.make28
-rw-r--r--ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp6
-rw-r--r--ydb/library/yql/providers/dq/ya.make8
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp4
-rw-r--r--ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp2
-rw-r--r--ydb/library/yql/providers/result/expr_nodes/ya.make56
-rw-r--r--ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.cpp12
-rw-r--r--ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h114
-rw-r--r--ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.json204
-rw-r--r--ydb/library/yql/providers/result/provider/ya.make26
-rw-r--r--ydb/library/yql/providers/result/provider/yql_result_provider.cpp248
-rw-r--r--ydb/library/yql/providers/result/provider/yql_result_provider.h2
-rw-r--r--ydb/library/yql/providers/result/ya.make6
-rw-r--r--ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp2
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp8
-rw-r--r--ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp2
-rw-r--r--ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp4
-rw-r--r--ydb/library/yql/public/decimal/ya.make4
-rw-r--r--ydb/library/yql/public/decimal/yql_decimal.cpp12
-rw-r--r--ydb/library/yql/public/decimal/yql_decimal.h98
-rw-r--r--ydb/library/yql/public/decimal/yql_decimal_serialize.cpp164
-rw-r--r--ydb/library/yql/public/decimal/yql_decimal_serialize.h24
-rw-r--r--ydb/library/yql/public/issue/ut/ya.make2
-rw-r--r--ydb/library/yql/public/issue/ya.make4
-rw-r--r--ydb/library/yql/public/issue/yql_issue.cpp28
-rw-r--r--ydb/library/yql/public/issue/yql_issue.h10
-rw-r--r--ydb/library/yql/public/issue/yql_issue_manager.cpp160
-rw-r--r--ydb/library/yql/public/issue/yql_issue_manager.h56
-rw-r--r--ydb/library/yql/public/issue/yql_issue_manager_ut.cpp182
-rw-r--r--ydb/library/yql/public/issue/yql_issue_message.cpp90
-rw-r--r--ydb/library/yql/public/issue/yql_issue_message.h12
-rw-r--r--ydb/library/yql/public/issue/yql_issue_ut.cpp14
-rw-r--r--ydb/library/yql/public/issue/yql_issue_utils.cpp228
-rw-r--r--ydb/library/yql/public/issue/yql_issue_utils.h62
-rw-r--r--ydb/library/yql/public/issue/yql_issue_utils_ut.cpp400
-rw-r--r--ydb/library/yql/public/udf/udf_helpers.cpp50
-rw-r--r--ydb/library/yql/public/udf/udf_helpers.h50
-rw-r--r--ydb/library/yql/public/udf/udf_type_builder.h16
-rw-r--r--ydb/library/yql/public/udf/udf_type_ops.h128
-rw-r--r--ydb/library/yql/public/udf/udf_value.h2
-rw-r--r--ydb/library/yql/public/udf/udf_value_inl.h28
-rw-r--r--ydb/library/yql/sql/settings/translation_settings.h4
-rw-r--r--ydb/library/yql/sql/v0/SQL.g4
-rw-r--r--ydb/library/yql/sql/v0/builtin.cpp2
-rw-r--r--ydb/library/yql/sql/v0/context.cpp92
-rw-r--r--ydb/library/yql/sql/v0/context.h10
-rw-r--r--ydb/library/yql/sql/v0/insert.cpp22
-rw-r--r--ydb/library/yql/sql/v0/join.cpp12
-rw-r--r--ydb/library/yql/sql/v0/node.cpp102
-rw-r--r--ydb/library/yql/sql/v0/node.h10
-rw-r--r--ydb/library/yql/sql/v0/query.cpp104
-rw-r--r--ydb/library/yql/sql/v0/select.cpp688
-rw-r--r--ydb/library/yql/sql/v0/sql.cpp306
-rw-r--r--ydb/library/yql/sql/v0/sql.h4
-rw-r--r--ydb/library/yql/sql/v0/sql_ut.cpp6
-rw-r--r--ydb/library/yql/sql/v1/SQLv1.g.in14
-rw-r--r--ydb/library/yql/sql/v1/builtin.cpp22
-rw-r--r--ydb/library/yql/sql/v1/context.cpp84
-rw-r--r--ydb/library/yql/sql/v1/context.h14
-rw-r--r--ydb/library/yql/sql/v1/insert.cpp20
-rw-r--r--ydb/library/yql/sql/v1/join.cpp12
-rw-r--r--ydb/library/yql/sql/v1/list_builtin.cpp2
-rw-r--r--ydb/library/yql/sql/v1/node.cpp110
-rw-r--r--ydb/library/yql/sql/v1/node.h34
-rw-r--r--ydb/library/yql/sql/v1/query.cpp124
-rw-r--r--ydb/library/yql/sql/v1/select.cpp894
-rw-r--r--ydb/library/yql/sql/v1/sql.cpp274
-rw-r--r--ydb/library/yql/sql/v1/sql.h4
-rw-r--r--ydb/library/yql/sql/v1/sql_ut.cpp62
-rw-r--r--ydb/library/yql/udfs/common/hyperscan/ya.make30
-rw-r--r--ydb/library/yql/udfs/common/math/math_ir.cpp10
-rw-r--r--ydb/library/yql/udfs/common/math/math_ir.h224
-rw-r--r--ydb/library/yql/udfs/common/math/math_udf.cpp104
-rw-r--r--ydb/library/yql/udfs/common/math/ya.make94
-rw-r--r--ydb/library/yql/udfs/common/string/string_udf.cpp108
-rw-r--r--ydb/library/yql/utils/backtrace/backtrace.cpp100
-rw-r--r--ydb/library/yql/utils/backtrace/backtrace.h4
-rw-r--r--ydb/library/yql/utils/debug_info.cpp16
-rw-r--r--ydb/library/yql/utils/failure_injector/failure_injector.cpp84
-rw-r--r--ydb/library/yql/utils/failure_injector/failure_injector.h40
-rw-r--r--ydb/library/yql/utils/failure_injector/failure_injector_ut.cpp122
-rw-r--r--ydb/library/yql/utils/fetch/fetch.cpp290
-rw-r--r--ydb/library/yql/utils/fetch/fetch.h40
-rw-r--r--ydb/library/yql/utils/fetch/ya.make22
-rw-r--r--ydb/library/yql/utils/log/context.cpp38
-rw-r--r--ydb/library/yql/utils/log/context.h60
-rw-r--r--ydb/library/yql/utils/log/log.cpp130
-rw-r--r--ydb/library/yql/utils/log/log.h36
-rw-r--r--ydb/library/yql/utils/log/log_component.h6
-rw-r--r--ydb/library/yql/utils/log/log_ut.cpp172
-rw-r--r--ydb/library/yql/utils/log/ya.make22
-rw-r--r--ydb/library/yql/utils/proc_alive.cpp74
-rw-r--r--ydb/library/yql/utils/proc_alive.h18
-rw-r--r--ydb/library/yql/utils/threading/async_semaphore.cpp150
-rw-r--r--ydb/library/yql/utils/threading/async_semaphore.h110
-rw-r--r--ydb/library/yql/utils/threading/ya.make18
-rw-r--r--ydb/library/yql/utils/ya.make4
502 files changed, 30095 insertions, 30095 deletions
diff --git a/build/conf/project_specific/yql_udf.conf b/build/conf/project_specific/yql_udf.conf
index badaf36687..f018412713 100644
--- a/build/conf/project_specific/yql_udf.conf
+++ b/build/conf/project_specific/yql_udf.conf
@@ -60,7 +60,7 @@ module YQL_UDF_TEST: PY2TEST {
DEPENDS(yql/tools/udf_resolver)
DEPENDS(yql/tools/yqlrun)
DATA(arcadia/ydb/library/yql/mount)
- DATA(arcadia/yql/cfg/tests)
+ DATA(arcadia/yql/cfg/tests)
}
### @usage: _ADD_YQL_UDF_DEPS()
diff --git a/library/cpp/binsaver/bin_saver.h b/library/cpp/binsaver/bin_saver.h
index 412424889f..f9938cc074 100644
--- a/library/cpp/binsaver/bin_saver.h
+++ b/library/cpp/binsaver/bin_saver.h
@@ -9,7 +9,7 @@
#include <util/generic/buffer.h>
#include <util/generic/list.h>
#include <util/generic/maybe.h>
-#include <util/generic/bitmap.h>
+#include <util/generic/bitmap.h>
#include <util/generic/variant.h>
#include <util/generic/ylimits.h>
#include <util/memory/blob.h>
@@ -426,31 +426,31 @@ public:
}
int Add(const chunk_id, TDynBitMap* pData) {
- if (IsReading()) {
- ui64 count = 0;
- Add(1, &count);
- pData->Clear();
- pData->Reserve(count * sizeof(TDynBitMap::TChunk) * 8);
- for (ui64 i = 0; i < count; ++i) {
- TDynBitMap::TChunk chunk = 0;
- Add(i + 1, &chunk);
- if (i > 0) {
- pData->LShift(8 * sizeof(TDynBitMap::TChunk));
- }
- pData->Or(chunk);
- }
- } else {
- ui64 count = pData->GetChunkCount();
- Add(1, &count);
- for (ui64 i = 0; i < count; ++i) {
- // Write in reverse order
- TDynBitMap::TChunk chunk = pData->GetChunks()[count - i - 1];
- Add(i + 1, &chunk);
- }
- }
- return 0;
- }
-
+ if (IsReading()) {
+ ui64 count = 0;
+ Add(1, &count);
+ pData->Clear();
+ pData->Reserve(count * sizeof(TDynBitMap::TChunk) * 8);
+ for (ui64 i = 0; i < count; ++i) {
+ TDynBitMap::TChunk chunk = 0;
+ Add(i + 1, &chunk);
+ if (i > 0) {
+ pData->LShift(8 * sizeof(TDynBitMap::TChunk));
+ }
+ pData->Or(chunk);
+ }
+ } else {
+ ui64 count = pData->GetChunkCount();
+ Add(1, &count);
+ for (ui64 i = 0; i < count; ++i) {
+ // Write in reverse order
+ TDynBitMap::TChunk chunk = pData->GetChunks()[count - i - 1];
+ Add(i + 1, &chunk);
+ }
+ }
+ return 0;
+ }
+
template <class TVariantClass>
struct TLoadFromTypeFromListHelper {
template <class T0, class... TTail>
diff --git a/library/cpp/codecs/codecs_registry.cpp b/library/cpp/codecs/codecs_registry.cpp
index 17d07062ab..bfc145bade 100644
--- a/library/cpp/codecs/codecs_registry.cpp
+++ b/library/cpp/codecs/codecs_registry.cpp
@@ -27,8 +27,8 @@ namespace NCodecs {
Y_VERIFY(!Registry.contains(name), "already has %s", name.data());
Registry[name] = fac;
}
- }
-
+ }
+
TCodecPtr TCodecRegistry::GetCodec(TStringBuf name) const {
using namespace NPrivate;
diff --git a/library/cpp/codecs/solar_codec.h b/library/cpp/codecs/solar_codec.h
index 7158ae7926..f0093f82f9 100644
--- a/library/cpp/codecs/solar_codec.h
+++ b/library/cpp/codecs/solar_codec.h
@@ -13,15 +13,15 @@ namespace NCodecs {
struct TVarIntTraits {
static const size_t MAX_VARINT32_BYTES = 5;
-
+
static void Write(ui32 value, TBuffer& b) {
while (value > 0x7F) {
b.Append(static_cast<ui8>(value) | 0x80);
value >>= 7;
}
b.Append(static_cast<ui8>(value) & 0x7F);
- }
-
+ }
+
static void Read(TStringBuf& r, ui32& value) {
ui32 result = 0;
for (ui32 count = 0; count < MAX_VARINT32_BYTES; ++count) {
@@ -34,22 +34,22 @@ namespace NCodecs {
} else if (Y_UNLIKELY(r.empty())) {
break;
}
- }
+ }
Y_ENSURE_EX(false, TCodecException() << "Bad data");
- }
+ }
};
-
+
struct TShortIntTraits {
static const size_t SHORTINT_SIZE_LIMIT = 0x8000;
-
+
Y_FORCE_INLINE static void Write(ui32 value, TBuffer& b) {
Y_ENSURE_EX(value < SHORTINT_SIZE_LIMIT, TCodecException() << "Bad write method");
if (value >= 0x80) {
b.Append(static_cast<ui8>(value >> 8) | 0x80);
}
b.Append(static_cast<ui8>(value));
- }
-
+ }
+
Y_FORCE_INLINE static void Read(TStringBuf& r, ui32& value) {
ui32 result = static_cast<ui8>(r[0]);
r.Skip(1);
@@ -59,9 +59,9 @@ namespace NCodecs {
r.Skip(1);
}
value = result;
- }
+ }
};
-
+
class TSolarCodec: public ICodec {
public:
static TStringBuf MyName8k() {
@@ -115,28 +115,28 @@ namespace NCodecs {
EncodeImpl<TVarIntTraits>(r, b);
return 0;
}
-
+
void Decode(TStringBuf r, TBuffer& b) const override {
DecodeImpl<TVarIntTraits>(r, b);
}
-
+
TString GetName() const override {
return ToString(MyName());
}
-
+
protected:
void DoLearn(ISequenceReader&) override;
void Save(IOutputStream*) const override;
void Load(IInputStream*) override;
-
+
Y_FORCE_INLINE TStringBuf SubStr(ui32 begoff, ui32 endoff) const {
return TStringBuf(Pool.Data() + begoff, endoff - begoff);
}
-
+
Y_FORCE_INLINE TStringBuf DoDecode(ui32 num) const {
return SubStr(Decoder[num - 1], Decoder[num]);
}
-
+
template <class TTraits>
Y_FORCE_INLINE void EncodeImpl(TStringBuf r, TBuffer& b) const {
b.Clear();
@@ -195,8 +195,8 @@ namespace NCodecs {
}
return 0;
- }
-
+ }
+
void Decode(TStringBuf r, TBuffer& b) const override {
if (CanUseShortInt()) {
DecodeImpl<TShortIntTraits>(r, b);
@@ -204,23 +204,23 @@ namespace NCodecs {
DecodeImpl<TVarIntTraits>(r, b);
}
}
-
+
TString GetName() const override {
if (CanUseShortInt()) {
return ToString(MyNameShortInt());
} else {
return ToString(MyName());
}
- }
+ }
};
-
+
class TSolarCodecShortInt: public TSolarCodec {
public:
explicit TSolarCodecShortInt(ui32 maxentries = 1 << 14, ui32 maxiter = 16, const NGreedyDict::TBuildSettings& s = NGreedyDict::TBuildSettings())
: TSolarCodec(maxentries, maxiter, s)
{
- }
-
+ }
+
ui8 /*free bits in last byte*/ Encode(TStringBuf r, TBuffer& b) const override {
EncodeImpl<TShortIntTraits>(r, b);
return 0;
@@ -229,16 +229,16 @@ namespace NCodecs {
void Decode(TStringBuf r, TBuffer& b) const override {
DecodeImpl<TShortIntTraits>(r, b);
}
-
+
TString GetName() const override {
return ToString(MyNameShortInt());
}
-
+
protected:
void Load(IInputStream* in) override {
TSolarCodec::Load(in);
Y_ENSURE_EX(CanUseShortInt(), TCodecException() << "Bad data");
}
};
-
+
}
diff --git a/library/cpp/codecs/ut/codecs_ut.cpp b/library/cpp/codecs/ut/codecs_ut.cpp
index caf6089aef..4bebbad08f 100644
--- a/library/cpp/codecs/ut/codecs_ut.cpp
+++ b/library/cpp/codecs/ut/codecs_ut.cpp
@@ -1281,9 +1281,9 @@ private:
}
TestCodec<TSolarCodec, true>(learn, test, new TSolarCodec(512, 8));
- TestCodec<TAdaptiveSolarCodec, false>(learn, test, new TAdaptiveSolarCodec(512, 8));
- TestCodec<TAdaptiveSolarCodec, true>(learn, test, new TAdaptiveSolarCodec(512, 8));
- TestCodec<TSolarCodecShortInt, true>(learn, test, new TSolarCodecShortInt(512, 8));
+ TestCodec<TAdaptiveSolarCodec, false>(learn, test, new TAdaptiveSolarCodec(512, 8));
+ TestCodec<TAdaptiveSolarCodec, true>(learn, test, new TAdaptiveSolarCodec(512, 8));
+ TestCodec<TSolarCodecShortInt, true>(learn, test, new TSolarCodecShortInt(512, 8));
}
{
@@ -1302,9 +1302,9 @@ private:
}
TestCodec<TSolarCodec, true>(learn, test, new TSolarCodec(512, 8));
- TestCodec<TAdaptiveSolarCodec, false>(learn, test, new TAdaptiveSolarCodec(512, 8));
- TestCodec<TAdaptiveSolarCodec, true>(learn, test, new TAdaptiveSolarCodec(512, 8));
- TestCodec<TSolarCodecShortInt, true>(learn, test, new TSolarCodecShortInt(512, 8));
+ TestCodec<TAdaptiveSolarCodec, false>(learn, test, new TAdaptiveSolarCodec(512, 8));
+ TestCodec<TAdaptiveSolarCodec, true>(learn, test, new TAdaptiveSolarCodec(512, 8));
+ TestCodec<TSolarCodecShortInt, true>(learn, test, new TSolarCodecShortInt(512, 8));
}
}
diff --git a/library/cpp/containers/comptrie/comptrie_impl.h b/library/cpp/containers/comptrie/comptrie_impl.h
index f41c38311a..068bed2e6c 100644
--- a/library/cpp/containers/comptrie/comptrie_impl.h
+++ b/library/cpp/containers/comptrie/comptrie_impl.h
@@ -182,40 +182,40 @@ namespace NCompactTrie {
// If there is a value associated with the symbol, makes the value pointer point to it,
// otherwise sets it to nullptr.
// Returns true if the symbol was succesfully found in the trie, false otherwise.
- template <typename TSymbol, class TPacker>
+ template <typename TSymbol, class TPacker>
Y_FORCE_INLINE bool Advance(const char*& datapos, const char* const dataend, const char*& value,
TSymbol label, TPacker packer) {
Y_ASSERT(datapos < dataend);
- char flags = MT_NEXT;
- for (int i = (int)ExtraBits<TSymbol>(); i >= 0; i -= 8) {
- flags = LeapByte(datapos, dataend, (char)(label >> i));
- if (!datapos) {
- return false; // no such arc
- }
-
+ char flags = MT_NEXT;
+ for (int i = (int)ExtraBits<TSymbol>(); i >= 0; i -= 8) {
+ flags = LeapByte(datapos, dataend, (char)(label >> i));
+ if (!datapos) {
+ return false; // no such arc
+ }
+
value = nullptr;
-
+
Y_ASSERT(datapos <= dataend);
- if ((flags & MT_FINAL)) {
- value = datapos;
- datapos += packer.SkipLeaf(datapos);
- }
-
- if (!(flags & MT_NEXT)) {
- if (i == 0) {
+ if ((flags & MT_FINAL)) {
+ value = datapos;
+ datapos += packer.SkipLeaf(datapos);
+ }
+
+ if (!(flags & MT_NEXT)) {
+ if (i == 0) {
datapos = nullptr;
- return true;
- }
- return false; // no further way
- }
-
- TraverseEpsilon(datapos);
- if (i == 0) { // last byte, and got a match
- return true;
- }
- }
-
- return false;
- }
-
+ return true;
+ }
+ return false; // no further way
+ }
+
+ TraverseEpsilon(datapos);
+ if (i == 0) { // last byte, and got a match
+ return true;
+ }
+ }
+
+ return false;
+ }
+
}
diff --git a/library/cpp/containers/comptrie/comptrie_trie.h b/library/cpp/containers/comptrie/comptrie_trie.h
index 40ec1e52b3..ffba10ae40 100644
--- a/library/cpp/containers/comptrie/comptrie_trie.h
+++ b/library/cpp/containers/comptrie/comptrie_trie.h
@@ -139,7 +139,7 @@ public:
// same as FindTails(&key, 1), a bit faster
// return false, if no arc with @label exists
- inline bool FindTails(TSymbol label, TCompactTrie<T, D, S>& res) const;
+ inline bool FindTails(TSymbol label, TCompactTrie<T, D, S>& res) const;
class TConstIterator {
private:
@@ -361,28 +361,28 @@ bool TCompactTrie<T, D, S>::FindTails(const TSymbol* key, size_t keylen, TCompac
return true;
}
- const char* datastart = DataHolder.AsCharPtr();
- const char* datapos = datastart;
- const char* const dataend = datapos + len;
-
+ const char* datastart = DataHolder.AsCharPtr();
+ const char* datapos = datastart;
+ const char* const dataend = datapos + len;
+
const TSymbol* keyend = key + keylen;
const char* value = nullptr;
-
+
while (key != keyend) {
T label = *(key++);
- if (!NCompactTrie::Advance(datapos, dataend, value, label, Packer))
- return false;
+ if (!NCompactTrie::Advance(datapos, dataend, value, label, Packer))
+ return false;
- if (key == keyend) {
- if (datapos) {
+ if (key == keyend) {
+ if (datapos) {
Y_ASSERT(datapos >= datastart);
res = TCompactTrie<T, D, S>(TBlob::NoCopy(datapos, dataend - datapos), value);
- } else {
- res = TCompactTrie<T, D, S>(value);
+ } else {
+ res = TCompactTrie<T, D, S>(value);
}
- return true;
- } else if (!datapos) {
- return false; // No further way
+ return true;
+ } else if (!datapos) {
+ return false; // No further way
}
}
@@ -390,7 +390,7 @@ bool TCompactTrie<T, D, S>::FindTails(const TSymbol* key, size_t keylen, TCompac
}
template <class T, class D, class S>
-inline bool TCompactTrie<T, D, S>::FindTails(TSymbol label, TCompactTrie<T, D, S>& res) const {
+inline bool TCompactTrie<T, D, S>::FindTails(TSymbol label, TCompactTrie<T, D, S>& res) const {
using namespace NCompactTrie;
const size_t len = DataHolder.Length();
@@ -402,17 +402,17 @@ inline bool TCompactTrie<T, D, S>::FindTails(TSymbol label, TCompactTrie<T, D, S
const char* datapos = datastart;
const char* value = nullptr;
- if (!NCompactTrie::Advance(datapos, dataend, value, label, Packer))
- return false;
+ if (!NCompactTrie::Advance(datapos, dataend, value, label, Packer))
+ return false;
- if (datapos) {
+ if (datapos) {
Y_ASSERT(datapos >= datastart);
- res = TCompactTrie<T, D, S>(TBlob::NoCopy(datapos, dataend - datapos), value);
- } else {
- res = TCompactTrie<T, D, S>(value);
+ res = TCompactTrie<T, D, S>(TBlob::NoCopy(datapos, dataend - datapos), value);
+ } else {
+ res = TCompactTrie<T, D, S>(value);
}
- return true;
+ return true;
}
template <class T, class D, class S>
diff --git a/library/cpp/containers/comptrie/comptrie_ut.cpp b/library/cpp/containers/comptrie/comptrie_ut.cpp
index 74bee09b5d..ea7df5c59d 100644
--- a/library/cpp/containers/comptrie/comptrie_ut.cpp
+++ b/library/cpp/containers/comptrie/comptrie_ut.cpp
@@ -96,7 +96,7 @@ private:
UNIT_TEST(TestSearchIterChar);
UNIT_TEST(TestSearchIterWchar);
UNIT_TEST(TestSearchIterWchar32)
-
+
UNIT_TEST(TestCopyAndAssignment);
UNIT_TEST(TestFirstSymbolIterator8);
@@ -152,9 +152,9 @@ private:
template <class TContainer>
void TestTrieWithContainers(const TVector<TUtf16String>& keys, const TVector<TContainer>& sampleData, TString methodName);
- template <typename TChar>
- void TestSearchIterImpl();
-
+ template <typename TChar>
+ void TestSearchIterImpl();
+
template <class TTrie>
void TestFirstSymbolIteratorForTrie(const TTrie& trie, const TStringBuf& narrowAnswers);
@@ -230,8 +230,8 @@ public:
void TestEmptyValueOutOfOrder();
void TestFindLongestPrefixWithEmptyValue();
- void TestSearchIterChar();
- void TestSearchIterWchar();
+ void TestSearchIterChar();
+ void TestSearchIterWchar();
void TestSearchIterWchar32();
void TestCopyAndAssignment();
@@ -1292,21 +1292,21 @@ void TCompactTrieTest::TestFindLongestPrefixWithEmptyValue() {
UNIT_ASSERT(value == 31415);
}
}
-
-template <typename TChar>
-struct TConvertKey {
+
+template <typename TChar>
+struct TConvertKey {
static inline TString Convert(const TStringBuf& key) {
return ToString(key);
- }
-};
-
-template <>
-struct TConvertKey<wchar16> {
+ }
+};
+
+template <>
+struct TConvertKey<wchar16> {
static inline TUtf16String Convert(const TStringBuf& key) {
- return UTF8ToWide(key);
- }
-};
-
+ return UTF8ToWide(key);
+ }
+};
+
template <>
struct TConvertKey<wchar32> {
static inline TUtf32String Convert(const TStringBuf& key) {
@@ -1314,62 +1314,62 @@ struct TConvertKey<wchar32> {
}
};
-template <class TSearchIter, class TKeyBuf>
-static void MoveIter(TSearchIter& iter, const TKeyBuf& key) {
- for (size_t i = 0; i < key.length(); ++i) {
- UNIT_ASSERT(iter.Advance(key[i]));
- }
-}
-
-template <typename TChar>
-void TCompactTrieTest::TestSearchIterImpl() {
- TBufferOutput buffer;
- {
- TCompactTrieBuilder<TChar, ui32> builder;
- TStringBuf data[] = {
+template <class TSearchIter, class TKeyBuf>
+static void MoveIter(TSearchIter& iter, const TKeyBuf& key) {
+ for (size_t i = 0; i < key.length(); ++i) {
+ UNIT_ASSERT(iter.Advance(key[i]));
+ }
+}
+
+template <typename TChar>
+void TCompactTrieTest::TestSearchIterImpl() {
+ TBufferOutput buffer;
+ {
+ TCompactTrieBuilder<TChar, ui32> builder;
+ TStringBuf data[] = {
TStringBuf("abaab"),
TStringBuf("abcdef"),
TStringBuf("abbbc"),
TStringBuf("bdfaa"),
- };
+ };
for (size_t i = 0; i < Y_ARRAY_SIZE(data); ++i) {
- builder.Add(TConvertKey<TChar>::Convert(data[i]), i + 1);
- }
- builder.Save(buffer);
- }
-
- TCompactTrie<TChar, ui32> trie(buffer.Buffer().Data(), buffer.Buffer().Size());
- ui32 value = 0;
+ builder.Add(TConvertKey<TChar>::Convert(data[i]), i + 1);
+ }
+ builder.Save(buffer);
+ }
+
+ TCompactTrie<TChar, ui32> trie(buffer.Buffer().Data(), buffer.Buffer().Size());
+ ui32 value = 0;
auto iter(MakeSearchIterator(trie));
MoveIter(iter, TConvertKey<TChar>::Convert(TStringBuf("abc")));
- UNIT_ASSERT(!iter.GetValue(&value));
-
+ UNIT_ASSERT(!iter.GetValue(&value));
+
iter = MakeSearchIterator(trie);
MoveIter(iter, TConvertKey<TChar>::Convert(TStringBuf("abbbc")));
- UNIT_ASSERT(iter.GetValue(&value));
- UNIT_ASSERT_EQUAL(value, 3);
-
+ UNIT_ASSERT(iter.GetValue(&value));
+ UNIT_ASSERT_EQUAL(value, 3);
+
iter = MakeSearchIterator(trie);
UNIT_ASSERT(iter.Advance(TConvertKey<TChar>::Convert(TStringBuf("bdfa"))));
- UNIT_ASSERT(!iter.GetValue(&value));
-
+ UNIT_ASSERT(!iter.GetValue(&value));
+
iter = MakeSearchIterator(trie);
UNIT_ASSERT(iter.Advance(TConvertKey<TChar>::Convert(TStringBuf("bdfaa"))));
- UNIT_ASSERT(iter.GetValue(&value));
- UNIT_ASSERT_EQUAL(value, 4);
-
+ UNIT_ASSERT(iter.GetValue(&value));
+ UNIT_ASSERT_EQUAL(value, 4);
+
UNIT_ASSERT(!MakeSearchIterator(trie).Advance(TChar('z')));
UNIT_ASSERT(!MakeSearchIterator(trie).Advance(TConvertKey<TChar>::Convert(TStringBuf("cdf"))));
UNIT_ASSERT(!MakeSearchIterator(trie).Advance(TConvertKey<TChar>::Convert(TStringBuf("abca"))));
-}
-
-void TCompactTrieTest::TestSearchIterChar() {
- TestSearchIterImpl<char>();
-}
-
-void TCompactTrieTest::TestSearchIterWchar() {
- TestSearchIterImpl<wchar16>();
-}
+}
+
+void TCompactTrieTest::TestSearchIterChar() {
+ TestSearchIterImpl<char>();
+}
+
+void TCompactTrieTest::TestSearchIterWchar() {
+ TestSearchIterImpl<wchar16>();
+}
void TCompactTrieTest::TestSearchIterWchar32() {
TestSearchIterImpl<wchar32>();
diff --git a/library/cpp/containers/comptrie/protopacker.h b/library/cpp/containers/comptrie/protopacker.h
index 3e15866dc5..1e3785d5b7 100644
--- a/library/cpp/containers/comptrie/protopacker.h
+++ b/library/cpp/containers/comptrie/protopacker.h
@@ -1,6 +1,6 @@
#pragma once
-#include <util/stream/mem.h>
+#include <util/stream/mem.h>
#include <util/ysaveload.h>
template <class Proto>
diff --git a/library/cpp/containers/sorted_vector/ut/ya.make b/library/cpp/containers/sorted_vector/ut/ya.make
index eb8a5b4bef..bc7d72600f 100644
--- a/library/cpp/containers/sorted_vector/ut/ya.make
+++ b/library/cpp/containers/sorted_vector/ut/ya.make
@@ -1,4 +1,4 @@
-UNITTEST_FOR(library/cpp/containers/sorted_vector)
+UNITTEST_FOR(library/cpp/containers/sorted_vector)
OWNER(udovichenko-r)
diff --git a/library/cpp/containers/ya.make b/library/cpp/containers/ya.make
index 4b1b315e6a..22d28fd753 100644
--- a/library/cpp/containers/ya.make
+++ b/library/cpp/containers/ya.make
@@ -54,8 +54,8 @@ RECURSE(
safe_vector
safe_vector/ut
segmented_pool_container
- sorted_vector
- sorted_vector/ut
+ sorted_vector
+ sorted_vector/ut
spars_ar
stack_array
stack_array/ut
diff --git a/library/cpp/ya.make b/library/cpp/ya.make
index 8c1193b007..68b374acaa 100644
--- a/library/cpp/ya.make
+++ b/library/cpp/ya.make
@@ -277,8 +277,8 @@ RECURSE(
pop_count
pop_count/benchmark
pop_count/ut
- presort
- presort/ut
+ presort
+ presort/ut
prob_counter
prob_counter/ut
progress_bar
@@ -356,8 +356,8 @@ RECURSE(
succinct_arrays
succinct_arrays/ut
svnversion
- telfinder
- telfinder/ut
+ telfinder
+ telfinder/ut
terminate_handler
terminate_handler/sample
testing
@@ -386,7 +386,7 @@ RECURSE(
udp
uilangdetect
uilangdetect/ut
- unicode
+ unicode
unistat
unistat/ut
uri
diff --git a/library/cpp/yson/node/node.cpp b/library/cpp/yson/node/node.cpp
index b39e070718..c832447d3b 100644
--- a/library/cpp/yson/node/node.cpp
+++ b/library/cpp/yson/node/node.cpp
@@ -878,9 +878,9 @@ bool operator==(const TNode& lhs, const TNode& rhs)
if (lhs.Attributes_) {
if (rhs.Attributes_) {
- if (*lhs.Attributes_ != *rhs.Attributes_) {
- return false;
- }
+ if (*lhs.Attributes_ != *rhs.Attributes_) {
+ return false;
+ }
} else {
return false;
}
diff --git a/library/cpp/yson/writer.cpp b/library/cpp/yson/writer.cpp
index 054459f9f5..d1dc7c672e 100644
--- a/library/cpp/yson/writer.cpp
+++ b/library/cpp/yson/writer.cpp
@@ -341,7 +341,7 @@ namespace NYson {
////////////////////////////////////////////////////////////////////////////////
void ReformatYsonStream(
- IInputStream* input,
+ IInputStream* input,
IOutputStream* output,
EYsonFormat format,
EYsonType type) {
diff --git a/library/cpp/yson/writer.h b/library/cpp/yson/writer.h
index 40f5d7d501..722ebe5e70 100644
--- a/library/cpp/yson/writer.h
+++ b/library/cpp/yson/writer.h
@@ -79,7 +79,7 @@ namespace NYson {
////////////////////////////////////////////////////////////////////////////////
void ReformatYsonStream(
- IInputStream* input,
+ IInputStream* input,
IOutputStream* output,
EYsonFormat format = EYsonFormat::Binary,
EYsonType type = ::NYson::EYsonType::Node);
diff --git a/tools/ya.make b/tools/ya.make
index 51a6b8b426..37d928586a 100644
--- a/tools/ya.make
+++ b/tools/ya.make
@@ -93,7 +93,7 @@ RECURSE(
printdom
printkeys
printlinks
- printphones
+ printphones
printreqs
printreqs/tests
printrichnode
diff --git a/util/generic/algorithm.h b/util/generic/algorithm.h
index badfb88993..6544d56fa2 100644
--- a/util/generic/algorithm.h
+++ b/util/generic/algorithm.h
@@ -388,11 +388,11 @@ static inline TO RemoveCopyIf(TI f, TI l, TO t, TP p) {
return std::remove_copy_if(f, l, t, p);
}
-template <class TI, class TO>
-static inline TO ReverseCopy(TI f, TI l, TO t) {
+template <class TI, class TO>
+static inline TO ReverseCopy(TI f, TI l, TO t) {
return std::reverse_copy(f, l, t);
-}
-
+}
+
template <class TI1, class TI2, class TO>
static inline TO SetUnion(TI1 f1, TI1 l1, TI2 f2, TI2 l2, TO p) {
return std::set_union(f1, l1, f2, l2, p);
diff --git a/util/generic/bitmap.h b/util/generic/bitmap.h
index f77d182460..d8611b9e69 100644
--- a/util/generic/bitmap.h
+++ b/util/generic/bitmap.h
@@ -3,16 +3,16 @@
#include "fwd.h"
#include "ptr.h"
#include "bitops.h"
-#include "typetraits.h"
+#include "typetraits.h"
#include "algorithm.h"
-#include "utility.h"
+#include "utility.h"
#include <util/system/yassert.h>
#include <util/system/defaults.h>
#include <util/str_stl.h>
#include <util/ysaveload.h>
-namespace NBitMapPrivate {
+namespace NBitMapPrivate {
// Returns number of bits set; result is in most significatnt byte
inline ui64 ByteSums(ui64 x) {
ui64 byteSums = x - ((x & 0xAAAAAAAAAAAAAAAAULL) >> 1);
@@ -33,34 +33,34 @@ namespace NBitMapPrivate {
struct TSanitizeMask {
static constexpr TChunkType Value = ~((~TChunkType(0)) << ExtraBits);
};
-
+
template <typename TChunkType>
struct TSanitizeMask<TChunkType, 0> {
static constexpr TChunkType Value = (TChunkType)~TChunkType(0u);
};
-
+
template <typename TTargetChunk, typename TSourceChunk>
struct TBigToSmallDataCopier {
static_assert(sizeof(TTargetChunk) < sizeof(TSourceChunk), "expect sizeof(TTargetChunk) < sizeof(TSourceChunk)");
static_assert(0 == sizeof(TSourceChunk) % sizeof(TTargetChunk), "expect 0 == sizeof(TSourceChunk) % sizeof(TTargetChunk)");
-
+
static constexpr size_t BLOCK_SIZE = sizeof(TSourceChunk) / sizeof(TTargetChunk);
-
+
union TCnv {
TSourceChunk BigData;
TTargetChunk SmallData[BLOCK_SIZE];
};
-
+
static inline void CopyChunk(TTargetChunk* target, TSourceChunk source) {
TCnv c;
c.BigData = source;
-#if defined(_big_endian_)
+#if defined(_big_endian_)
::ReverseCopy(c.SmallData, c.SmallData + Y_ARRAY_SIZE(c.SmallData), target);
-#else
+#else
::Copy(c.SmallData, c.SmallData + Y_ARRAY_SIZE(c.SmallData), target);
-#endif
+#endif
}
-
+
static inline void Copy(TTargetChunk* target, size_t targetSize, const TSourceChunk* source, size_t sourceSize) {
Y_ASSERT(targetSize >= sourceSize * BLOCK_SIZE);
if (targetSize > sourceSize * BLOCK_SIZE) {
@@ -69,43 +69,43 @@ namespace NBitMapPrivate {
for (size_t i = 0; i < sourceSize; ++i) {
CopyChunk(target + i * BLOCK_SIZE, source[i]);
}
- }
+ }
};
-
+
template <typename TTargetChunk, typename TSourceChunk>
struct TSmallToBigDataCopier {
static_assert(sizeof(TTargetChunk) > sizeof(TSourceChunk), "expect sizeof(TTargetChunk) > sizeof(TSourceChunk)");
static_assert(0 == sizeof(TTargetChunk) % sizeof(TSourceChunk), "expect 0 == sizeof(TTargetChunk) % sizeof(TSourceChunk)");
-
+
static constexpr size_t BLOCK_SIZE = sizeof(TTargetChunk) / sizeof(TSourceChunk);
-
+
union TCnv {
TSourceChunk SmallData[BLOCK_SIZE];
TTargetChunk BigData;
};
-
+
static inline TTargetChunk CopyFullChunk(const TSourceChunk* source) {
TCnv c;
-#if defined(_big_endian_)
+#if defined(_big_endian_)
::ReverseCopy(source, source + BLOCK_SIZE, c.SmallData);
-#else
+#else
::Copy(source, source + BLOCK_SIZE, c.SmallData);
-#endif
+#endif
return c.BigData;
}
-
+
static inline TTargetChunk CopyPartChunk(const TSourceChunk* source, size_t count) {
Y_ASSERT(count <= BLOCK_SIZE);
TCnv c;
c.BigData = 0;
-#if defined(_big_endian_)
+#if defined(_big_endian_)
::ReverseCopy(source, source + count, c.SmallData);
-#else
+#else
::Copy(source, source + count, c.SmallData);
-#endif
+#endif
return c.BigData;
}
-
+
static inline void Copy(TTargetChunk* target, size_t targetSize, const TSourceChunk* source, size_t sourceSize) {
Y_ASSERT(targetSize * BLOCK_SIZE >= sourceSize);
if (targetSize * BLOCK_SIZE > sourceSize) {
@@ -118,9 +118,9 @@ namespace NBitMapPrivate {
if (0 != sourceSize % BLOCK_SIZE) {
target[i] = CopyPartChunk(source + i * BLOCK_SIZE, sourceSize % BLOCK_SIZE);
}
- }
+ }
};
-
+
template <typename TChunk>
struct TUniformDataCopier {
static inline void Copy(TChunk* target, size_t targetSize, const TChunk* source, size_t sourceSize) {
@@ -131,91 +131,91 @@ namespace NBitMapPrivate {
for (size_t i = sourceSize; i < targetSize; ++i) {
target[i] = 0;
}
- }
+ }
};
-
+
template <typename TFirst, typename TSecond>
struct TIsSmaller {
enum {
Result = sizeof(TFirst) < sizeof(TSecond)
};
- };
-
+ };
+
template <typename TTargetChunk, typename TSourceChunk>
struct TDataCopier: public std::conditional_t<std::is_same<TTargetChunk, TSourceChunk>::value, TUniformDataCopier<TTargetChunk>, std::conditional_t<TIsSmaller<TTargetChunk, TSourceChunk>::Result, TBigToSmallDataCopier<TTargetChunk, TSourceChunk>, TSmallToBigDataCopier<TTargetChunk, TSourceChunk>>> {
};
-
+
template <typename TTargetChunk, typename TSourceChunk>
inline void CopyData(TTargetChunk* target, size_t targetSize, const TSourceChunk* source, size_t sourceSize) {
TDataCopier<TTargetChunk, TSourceChunk>::Copy(target, targetSize, source, sourceSize);
}
-
+
template <size_t BitCount, typename TChunkType>
struct TFixedStorage {
using TChunk = TChunkType;
-
+
static constexpr size_t Size = (BitCount + 8 * sizeof(TChunk) - 1) / (8 * sizeof(TChunk));
-
+
TChunk Data[Size];
-
+
TFixedStorage() {
Zero(Data);
}
-
+
TFixedStorage(const TFixedStorage<BitCount, TChunkType>& st) {
for (size_t i = 0; i < Size; ++i) {
Data[i] = st.Data[i];
}
}
-
+
template <typename TOtherChunk>
TFixedStorage(const TOtherChunk* data, size_t size) {
Y_VERIFY(Size * sizeof(TChunk) >= size * sizeof(TOtherChunk), "Exceeding bitmap storage capacity");
CopyData(Data, Size, data, size);
- }
-
+ }
+
Y_FORCE_INLINE void Swap(TFixedStorage<BitCount, TChunkType>& st) {
for (size_t i = 0; i < Size; ++i) {
DoSwap(Data[i], st.Data[i]);
}
}
-
+
Y_FORCE_INLINE static constexpr size_t GetBitCapacity() noexcept {
return BitCount;
- }
-
+ }
+
Y_FORCE_INLINE static constexpr size_t GetChunkCapacity() noexcept {
return Size;
}
-
+
// Returns true if the resulting storage capacity is enough to fit the requested size
Y_FORCE_INLINE static constexpr bool ExpandBitSize(const size_t bitSize) noexcept {
return bitSize <= BitCount;
}
-
+
Y_FORCE_INLINE void Sanitize() {
Data[Size - 1] &= TSanitizeMask<TChunk, BitCount % (8 * sizeof(TChunk))>::Value;
}
};
-
+
// Dynamically expanded storage.
// It uses "on stack" realization with no allocation for one chunk spaces
template <typename TChunkType>
struct TDynamicStorage {
using TChunk = TChunkType;
-
+
size_t Size;
TChunk StackData;
TArrayHolder<TChunk> ArrayData;
TChunk* Data;
-
+
TDynamicStorage()
: Size(1)
, StackData(0)
, Data(&StackData)
{
}
-
+
TDynamicStorage(const TDynamicStorage<TChunk>& st)
: Size(1)
, StackData(0)
@@ -229,7 +229,7 @@ namespace NBitMapPrivate {
Data[i] = 0;
}
}
-
+
template <typename TOtherChunk>
TDynamicStorage(const TOtherChunk* data, size_t size)
: Size(1)
@@ -238,7 +238,7 @@ namespace NBitMapPrivate {
{
ExpandBitSize(size * sizeof(TOtherChunk) * 8, false);
CopyData(Data, Size, data, size);
- }
+ }
Y_FORCE_INLINE void Swap(TDynamicStorage<TChunkType>& st) {
DoSwap(Size, st.Size);
@@ -246,20 +246,20 @@ namespace NBitMapPrivate {
DoSwap(ArrayData, st.ArrayData);
Data = 1 == Size ? &StackData : ArrayData.Get();
st.Data = 1 == st.Size ? &st.StackData : st.ArrayData.Get();
- }
-
+ }
+
Y_FORCE_INLINE size_t GetBitCapacity() const {
return Size * 8 * sizeof(TChunk);
}
-
+
Y_FORCE_INLINE size_t GetChunkCapacity() const {
return Size;
}
-
+
// Returns true if the resulting storage capacity is enough to fit the requested size
Y_FORCE_INLINE bool ExpandSize(size_t size, bool keepData = true) {
if (size > Size) {
- size = Max(size, Size * 2);
+ size = Max(size, Size * 2);
TArrayHolder<TChunk> newData(new TChunk[size]);
if (keepData) {
for (size_t i = 0; i < Size; ++i) {
@@ -268,140 +268,140 @@ namespace NBitMapPrivate {
for (size_t i = Size; i < size; ++i) {
newData[i] = 0;
}
- }
+ }
DoSwap(ArrayData, newData);
Data = ArrayData.Get();
Size = size;
- }
+ }
return true;
- }
+ }
Y_FORCE_INLINE bool ExpandBitSize(size_t bitSize, bool keepData = true) {
return ExpandSize((bitSize + 8 * sizeof(TChunk) - 1) / (8 * sizeof(TChunk)), keepData);
}
-
+
Y_FORCE_INLINE void Sanitize() {
}
};
-
+
template <size_t num>
struct TDivCount {
static constexpr size_t Value = 1 + TDivCount<(num >> 1)>::Value;
};
-
+
template <>
struct TDivCount<0> {
static constexpr size_t Value = 0;
};
-
+
}
-
-template <size_t BitCount, typename TChunkType>
-struct TFixedBitMapTraits {
+
+template <size_t BitCount, typename TChunkType>
+struct TFixedBitMapTraits {
using TChunk = TChunkType;
using TStorage = NBitMapPrivate::TFixedStorage<BitCount, TChunkType>;
-};
-
-template <typename TChunkType>
-struct TDynamicBitMapTraits {
+};
+
+template <typename TChunkType>
+struct TDynamicBitMapTraits {
using TChunk = TChunkType;
using TStorage = NBitMapPrivate::TDynamicStorage<TChunkType>;
-};
-
-template <class TTraits>
-class TBitMapOps {
+};
+
+template <class TTraits>
+class TBitMapOps {
public:
using TChunk = typename TTraits::TChunk;
using TThis = TBitMapOps<TTraits>;
-
-private:
+
+private:
static_assert(std::is_unsigned<TChunk>::value, "expect std::is_unsigned<TChunk>::value");
-
+
static constexpr size_t BitsPerChunk = 8 * sizeof(TChunk);
static constexpr TChunk ModMask = static_cast<TChunk>(BitsPerChunk - 1);
static constexpr size_t DivCount = NBitMapPrivate::TDivCount<BitsPerChunk>::Value - 1;
static constexpr TChunk FullChunk = (TChunk)~TChunk(0);
-
+
template <class>
friend class TBitMapOps;
-
+
using TStorage = typename TTraits::TStorage;
-
- // The smallest unsigned type, which can be used in bit ops
+
+ // The smallest unsigned type, which can be used in bit ops
using TIntType = std::conditional_t<sizeof(TChunk) < sizeof(unsigned int), unsigned int, TChunk>;
-
- TStorage Mask;
-
-public:
- class TReference {
- private:
- friend class TBitMapOps<TTraits>;
-
- TChunk* Chunk;
- size_t Offset;
-
- TReference(TChunk* c, size_t offset)
- : Chunk(c)
- , Offset(offset)
- {
- }
-
- public:
+
+ TStorage Mask;
+
+public:
+ class TReference {
+ private:
+ friend class TBitMapOps<TTraits>;
+
+ TChunk* Chunk;
+ size_t Offset;
+
+ TReference(TChunk* c, size_t offset)
+ : Chunk(c)
+ , Offset(offset)
+ {
+ }
+
+ public:
~TReference() = default;
-
+
Y_FORCE_INLINE TReference& operator=(bool val) {
- if (val)
- *Chunk |= static_cast<TChunk>(1) << Offset;
- else
+ if (val)
+ *Chunk |= static_cast<TChunk>(1) << Offset;
+ else
*Chunk &= ~(static_cast<TChunk>(1) << Offset);
-
- return *this;
- }
-
+
+ return *this;
+ }
+
Y_FORCE_INLINE TReference& operator=(const TReference& ref) {
- if (ref)
- *Chunk |= static_cast<TChunk>(1) << Offset;
- else
- *Chunk &= ~(static_cast<TChunk>(1) << Offset);
-
- return *this;
- }
-
+ if (ref)
+ *Chunk |= static_cast<TChunk>(1) << Offset;
+ else
+ *Chunk &= ~(static_cast<TChunk>(1) << Offset);
+
+ return *this;
+ }
+
Y_FORCE_INLINE bool operator~() const {
- return 0 == (*Chunk & (static_cast<TChunk>(1) << Offset));
- }
-
+ return 0 == (*Chunk & (static_cast<TChunk>(1) << Offset));
+ }
+
Y_FORCE_INLINE operator bool() const {
- return 0 != (*Chunk & (static_cast<TChunk>(1) << Offset));
- }
-
+ return 0 != (*Chunk & (static_cast<TChunk>(1) << Offset));
+ }
+
Y_FORCE_INLINE TReference& Flip() {
- *Chunk ^= static_cast<TChunk>(1) << Offset;
- return *this;
- }
- };
-
-private:
- struct TSetOp {
+ *Chunk ^= static_cast<TChunk>(1) << Offset;
+ return *this;
+ }
+ };
+
+private:
+ struct TSetOp {
static constexpr TChunk Op(const TChunk src, const TChunk mask) noexcept {
- return src | mask;
- }
- };
-
- struct TResetOp {
+ return src | mask;
+ }
+ };
+
+ struct TResetOp {
static constexpr TChunk Op(const TChunk src, const TChunk mask) noexcept {
- return src & ~mask;
- }
- };
-
- template <class TUpdateOp>
- void UpdateRange(size_t start, size_t end) {
- const size_t startChunk = start >> DivCount;
- const size_t startBitOffset = start & ModMask;
-
- const size_t endChunk = end >> DivCount;
- const size_t endBitOffset = end & ModMask;
-
+ return src & ~mask;
+ }
+ };
+
+ template <class TUpdateOp>
+ void UpdateRange(size_t start, size_t end) {
+ const size_t startChunk = start >> DivCount;
+ const size_t startBitOffset = start & ModMask;
+
+ const size_t endChunk = end >> DivCount;
+ const size_t endBitOffset = end & ModMask;
+
size_t bitOffset = startBitOffset;
for (size_t chunk = startChunk; chunk <= endChunk; ++chunk) {
TChunk updateMask = FullChunk << bitOffset;
@@ -409,76 +409,76 @@ private:
updateMask ^= FullChunk << endBitOffset;
if (!updateMask)
break;
- }
+ }
Mask.Data[chunk] = TUpdateOp::Op(Mask.Data[chunk], updateMask);
bitOffset = 0;
- }
- }
-
-public:
+ }
+ }
+
+public:
TBitMapOps() = default;
TBitMapOps(TChunk val) {
- Mask.Data[0] = val;
- Mask.Sanitize();
+ Mask.Data[0] = val;
+ Mask.Sanitize();
}
TBitMapOps(const TThis&) = default;
- template <class T>
- TBitMapOps(const TBitMapOps<T>& bitmap)
- : Mask(bitmap.Mask.Data, bitmap.Mask.GetChunkCapacity())
- {
- Mask.Sanitize();
+ template <class T>
+ TBitMapOps(const TBitMapOps<T>& bitmap)
+ : Mask(bitmap.Mask.Data, bitmap.Mask.GetChunkCapacity())
+ {
+ Mask.Sanitize();
}
- template <class T>
+ template <class T>
Y_FORCE_INLINE bool operator==(const TBitMapOps<T>& bitmap) const {
- return Equal(bitmap);
- }
-
+ return Equal(bitmap);
+ }
+
Y_FORCE_INLINE TThis& operator=(const TThis& bitmap) {
- if (this != &bitmap) {
- TThis bm(bitmap);
- Swap(bm);
- }
- return *this;
- }
-
- template <class T>
+ if (this != &bitmap) {
+ TThis bm(bitmap);
+ Swap(bm);
+ }
+ return *this;
+ }
+
+ template <class T>
Y_FORCE_INLINE TThis& operator=(const TBitMapOps<T>& bitmap) {
- TThis bm(bitmap);
- Swap(bm);
- return *this;
- }
-
- template <class T>
+ TThis bm(bitmap);
+ Swap(bm);
+ return *this;
+ }
+
+ template <class T>
Y_FORCE_INLINE TThis& operator&=(const TBitMapOps<T>& bitmap) {
- return And(bitmap);
- }
-
+ return And(bitmap);
+ }
+
Y_FORCE_INLINE TThis& operator&=(const TChunk& val) {
- return And(val);
- }
-
- template <class T>
+ return And(val);
+ }
+
+ template <class T>
Y_FORCE_INLINE TThis& operator|=(const TBitMapOps<T>& bitmap) {
- return Or(bitmap);
- }
-
+ return Or(bitmap);
+ }
+
Y_FORCE_INLINE TThis& operator|=(const TChunk& val) {
- return Or(val);
- }
-
- template <class T>
+ return Or(val);
+ }
+
+ template <class T>
Y_FORCE_INLINE TThis& operator^=(const TBitMapOps<T>& bitmap) {
- return Xor(bitmap);
- }
-
+ return Xor(bitmap);
+ }
+
Y_FORCE_INLINE TThis& operator^=(const TChunk& val) {
- return Xor(val);
- }
-
+ return Xor(val);
+ }
+
template <class T>
Y_FORCE_INLINE TThis& operator-=(const TBitMapOps<T>& bitmap) {
return SetDifference(bitmap);
@@ -490,81 +490,81 @@ public:
Y_FORCE_INLINE TThis& operator<<=(size_t pos) {
return LShift(pos);
- }
-
+ }
+
Y_FORCE_INLINE TThis& operator>>=(size_t pos) {
return RShift(pos);
- }
-
+ }
+
Y_FORCE_INLINE TThis operator<<(size_t pos) const {
- return TThis(*this).LShift(pos);
- }
-
+ return TThis(*this).LShift(pos);
+ }
+
Y_FORCE_INLINE TThis operator>>(size_t pos) const {
- return TThis(*this).RShift(pos);
- }
-
+ return TThis(*this).RShift(pos);
+ }
+
Y_FORCE_INLINE bool operator[](size_t pos) const {
- return Get(pos);
- }
-
+ return Get(pos);
+ }
+
Y_FORCE_INLINE TReference operator[](size_t pos) {
const bool fitStorage = Mask.ExpandBitSize(pos + 1);
Y_ASSERT(fitStorage);
- return TReference(&Mask.Data[pos >> DivCount], ModMask & pos);
- }
-
+ return TReference(&Mask.Data[pos >> DivCount], ModMask & pos);
+ }
+
Y_FORCE_INLINE void Swap(TThis& bitmap) {
- DoSwap(Mask, bitmap.Mask);
- }
-
+ DoSwap(Mask, bitmap.Mask);
+ }
+
Y_FORCE_INLINE TThis& Set(size_t pos) {
const bool fitStorage = Mask.ExpandBitSize(pos + 1);
Y_ASSERT(fitStorage);
- Mask.Data[pos >> DivCount] |= static_cast<TChunk>(1) << (pos & ModMask);
- return *this;
- }
-
- // Fills the specified [start, end) bit range by the 1. Other bits are kept unchanged
- TThis& Set(size_t start, size_t end) {
+ Mask.Data[pos >> DivCount] |= static_cast<TChunk>(1) << (pos & ModMask);
+ return *this;
+ }
+
+ // Fills the specified [start, end) bit range by the 1. Other bits are kept unchanged
+ TThis& Set(size_t start, size_t end) {
Y_ASSERT(start <= end);
- if (start < end) {
- Reserve(end);
- UpdateRange<TSetOp>(start, end);
- }
- return *this;
- }
-
+ if (start < end) {
+ Reserve(end);
+ UpdateRange<TSetOp>(start, end);
+ }
+ return *this;
+ }
+
Y_FORCE_INLINE TThis& Reset(size_t pos) {
- if ((pos >> DivCount) < Mask.GetChunkCapacity()) {
- Mask.Data[pos >> DivCount] &= ~(static_cast<TChunk>(1) << (pos & ModMask));
- }
- return *this;
- }
-
- // Clears the specified [start, end) bit range. Other bits are kept unchanged
- TThis& Reset(size_t start, size_t end) {
+ if ((pos >> DivCount) < Mask.GetChunkCapacity()) {
+ Mask.Data[pos >> DivCount] &= ~(static_cast<TChunk>(1) << (pos & ModMask));
+ }
+ return *this;
+ }
+
+ // Clears the specified [start, end) bit range. Other bits are kept unchanged
+ TThis& Reset(size_t start, size_t end) {
Y_ASSERT(start <= end);
- if (start < end && (start >> DivCount) < Mask.GetChunkCapacity()) {
+ if (start < end && (start >> DivCount) < Mask.GetChunkCapacity()) {
UpdateRange<TResetOp>(start, Min(end, Mask.GetBitCapacity()));
- }
- return *this;
- }
-
+ }
+ return *this;
+ }
+
Y_FORCE_INLINE TThis& Flip(size_t pos) {
const bool fitStorage = Mask.ExpandBitSize(pos + 1);
Y_ASSERT(fitStorage);
- Mask.Data[pos >> DivCount] ^= static_cast<TChunk>(1) << (pos & ModMask);
- return *this;
- }
-
+ Mask.Data[pos >> DivCount] ^= static_cast<TChunk>(1) << (pos & ModMask);
+ return *this;
+ }
+
Y_FORCE_INLINE bool Get(size_t pos) const {
- if ((pos >> DivCount) < Mask.GetChunkCapacity()) {
- return Mask.Data[pos >> DivCount] & (static_cast<TChunk>(1) << (pos & ModMask));
- }
- return false;
- }
-
+ if ((pos >> DivCount) < Mask.GetChunkCapacity()) {
+ return Mask.Data[pos >> DivCount] & (static_cast<TChunk>(1) << (pos & ModMask));
+ }
+ return false;
+ }
+
template <class TTo>
void Export(size_t pos, TTo& to) const {
static_assert(std::is_unsigned<TTo>::value, "expect std::is_unsigned<TTo>::value");
@@ -593,152 +593,152 @@ public:
}
Y_FORCE_INLINE TThis& Push(bool val) {
- LShift(1);
- return val ? Set(0) : *this;
+ LShift(1);
+ return val ? Set(0) : *this;
}
Y_FORCE_INLINE bool Pop() {
- bool val = Get(0);
- return RShift(1), val;
+ bool val = Get(0);
+ return RShift(1), val;
}
- // Clear entire bitmap. Current capacity is kept unchanged
+ // Clear entire bitmap. Current capacity is kept unchanged
Y_FORCE_INLINE TThis& Clear() {
- for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i) {
- Mask.Data[i] = 0;
- }
- return *this;
+ for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i) {
+ Mask.Data[i] = 0;
+ }
+ return *this;
}
- // Returns bits capacity
+ // Returns bits capacity
Y_FORCE_INLINE constexpr size_t Size() const noexcept {
- return Mask.GetBitCapacity();
- }
-
+ return Mask.GetBitCapacity();
+ }
+
Y_FORCE_INLINE void Reserve(size_t bitCount) {
Y_VERIFY(Mask.ExpandBitSize(bitCount), "Exceeding bitmap storage capacity");
}
Y_FORCE_INLINE size_t ValueBitCount() const {
- size_t nonZeroChunk = Mask.GetChunkCapacity() - 1;
- while (nonZeroChunk != 0 && !Mask.Data[nonZeroChunk])
- --nonZeroChunk;
- return nonZeroChunk || Mask.Data[nonZeroChunk]
+ size_t nonZeroChunk = Mask.GetChunkCapacity() - 1;
+ while (nonZeroChunk != 0 && !Mask.Data[nonZeroChunk])
+ --nonZeroChunk;
+ return nonZeroChunk || Mask.Data[nonZeroChunk]
? nonZeroChunk * BitsPerChunk + GetValueBitCount(TIntType(Mask.Data[nonZeroChunk]))
: 0;
- }
-
+ }
+
Y_PURE_FUNCTION Y_FORCE_INLINE bool Empty() const {
- for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i)
- if (Mask.Data[i])
+ for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i)
+ if (Mask.Data[i])
return false;
return true;
}
- bool HasAny(const TThis& bitmap) const {
- for (size_t i = 0; i < Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity()); ++i) {
- if (0 != (Mask.Data[i] & bitmap.Mask.Data[i])) {
- return true;
- }
- }
- return false;
- }
-
- template <class T>
+ bool HasAny(const TThis& bitmap) const {
+ for (size_t i = 0; i < Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity()); ++i) {
+ if (0 != (Mask.Data[i] & bitmap.Mask.Data[i])) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ template <class T>
Y_FORCE_INLINE bool HasAny(const TBitMapOps<T>& bitmap) const {
- return HasAny(TThis(bitmap));
- }
-
+ return HasAny(TThis(bitmap));
+ }
+
Y_FORCE_INLINE bool HasAny(const TChunk& val) const {
- return 0 != (Mask.Data[0] & val);
- }
-
- bool HasAll(const TThis& bitmap) const {
- for (size_t i = 0; i < Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity()); ++i) {
- if (bitmap.Mask.Data[i] != (Mask.Data[i] & bitmap.Mask.Data[i])) {
- return false;
- }
- }
- for (size_t i = Mask.GetChunkCapacity(); i < bitmap.Mask.GetChunkCapacity(); ++i) {
- if (bitmap.Mask.Data[i] != 0) {
- return false;
- }
- }
- return true;
- }
-
- template <class T>
+ return 0 != (Mask.Data[0] & val);
+ }
+
+ bool HasAll(const TThis& bitmap) const {
+ for (size_t i = 0; i < Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity()); ++i) {
+ if (bitmap.Mask.Data[i] != (Mask.Data[i] & bitmap.Mask.Data[i])) {
+ return false;
+ }
+ }
+ for (size_t i = Mask.GetChunkCapacity(); i < bitmap.Mask.GetChunkCapacity(); ++i) {
+ if (bitmap.Mask.Data[i] != 0) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ template <class T>
Y_FORCE_INLINE bool HasAll(const TBitMapOps<T>& bitmap) const {
- return HasAll(TThis(bitmap));
- }
-
+ return HasAll(TThis(bitmap));
+ }
+
Y_FORCE_INLINE bool HasAll(const TChunk& val) const {
- return (Mask.Data[0] & val) == val;
- }
-
- TThis& And(const TThis& bitmap) {
- // Don't expand capacity here, because resulting bits in positions,
- // which are greater then size of one of these bitmaps, will be zero
- for (size_t i = 0; i < Min(bitmap.Mask.GetChunkCapacity(), Mask.GetChunkCapacity()); ++i)
- Mask.Data[i] &= bitmap.Mask.Data[i];
- // Clear bits if current bitmap size is greater than AND-ed one
- for (size_t i = bitmap.Mask.GetChunkCapacity(); i < Mask.GetChunkCapacity(); ++i)
- Mask.Data[i] = 0;
- return *this;
- }
-
- template <class T>
+ return (Mask.Data[0] & val) == val;
+ }
+
+ TThis& And(const TThis& bitmap) {
+ // Don't expand capacity here, because resulting bits in positions,
+ // which are greater then size of one of these bitmaps, will be zero
+ for (size_t i = 0; i < Min(bitmap.Mask.GetChunkCapacity(), Mask.GetChunkCapacity()); ++i)
+ Mask.Data[i] &= bitmap.Mask.Data[i];
+ // Clear bits if current bitmap size is greater than AND-ed one
+ for (size_t i = bitmap.Mask.GetChunkCapacity(); i < Mask.GetChunkCapacity(); ++i)
+ Mask.Data[i] = 0;
+ return *this;
+ }
+
+ template <class T>
Y_FORCE_INLINE TThis& And(const TBitMapOps<T>& bitmap) {
- return And(TThis(bitmap));
+ return And(TThis(bitmap));
}
Y_FORCE_INLINE TThis& And(const TChunk& val) {
- Mask.Data[0] &= val;
- for (size_t i = 1; i < Mask.GetChunkCapacity(); ++i)
- Mask.Data[i] = 0;
- return *this;
- }
-
- TThis& Or(const TThis& bitmap) {
- const size_t valueBitCount = bitmap.ValueBitCount();
- if (valueBitCount) {
- // Memory optimization: expand size only for non-zero bits
- Reserve(valueBitCount);
- for (size_t i = 0; i < Min(bitmap.Mask.GetChunkCapacity(), Mask.GetChunkCapacity()); ++i)
- Mask.Data[i] |= bitmap.Mask.Data[i];
- }
- return *this;
- }
-
- template <class T>
+ Mask.Data[0] &= val;
+ for (size_t i = 1; i < Mask.GetChunkCapacity(); ++i)
+ Mask.Data[i] = 0;
+ return *this;
+ }
+
+ TThis& Or(const TThis& bitmap) {
+ const size_t valueBitCount = bitmap.ValueBitCount();
+ if (valueBitCount) {
+ // Memory optimization: expand size only for non-zero bits
+ Reserve(valueBitCount);
+ for (size_t i = 0; i < Min(bitmap.Mask.GetChunkCapacity(), Mask.GetChunkCapacity()); ++i)
+ Mask.Data[i] |= bitmap.Mask.Data[i];
+ }
+ return *this;
+ }
+
+ template <class T>
Y_FORCE_INLINE TThis& Or(const TBitMapOps<T>& bitmap) {
- return Or(TThis(bitmap));
- }
-
+ return Or(TThis(bitmap));
+ }
+
Y_FORCE_INLINE TThis& Or(const TChunk& val) {
- Mask.Data[0] |= val;
- Mask.Sanitize();
- return *this;
- }
-
- TThis& Xor(const TThis& bitmap) {
+ Mask.Data[0] |= val;
+ Mask.Sanitize();
+ return *this;
+ }
+
+ TThis& Xor(const TThis& bitmap) {
Reserve(bitmap.Size());
- for (size_t i = 0; i < bitmap.Mask.GetChunkCapacity(); ++i)
- Mask.Data[i] ^= bitmap.Mask.Data[i];
- return *this;
- }
-
- template <class T>
+ for (size_t i = 0; i < bitmap.Mask.GetChunkCapacity(); ++i)
+ Mask.Data[i] ^= bitmap.Mask.Data[i];
+ return *this;
+ }
+
+ template <class T>
Y_FORCE_INLINE TThis& Xor(const TBitMapOps<T>& bitmap) {
- return Xor(TThis(bitmap));
- }
-
+ return Xor(TThis(bitmap));
+ }
+
Y_FORCE_INLINE TThis& Xor(const TChunk& val) {
- Mask.Data[0] ^= val;
- Mask.Sanitize();
- return *this;
- }
-
+ Mask.Data[0] ^= val;
+ Mask.Sanitize();
+ return *this;
+ }
+
TThis& SetDifference(const TThis& bitmap) {
for (size_t i = 0; i < Min(bitmap.Mask.GetChunkCapacity(), Mask.GetChunkCapacity()); ++i)
Mask.Data[i] &= ~bitmap.Mask.Data[i];
@@ -756,204 +756,204 @@ public:
}
Y_FORCE_INLINE TThis& Flip() {
- for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i)
- Mask.Data[i] = ~Mask.Data[i];
- Mask.Sanitize();
- return *this;
- }
-
- TThis& LShift(size_t shift) {
- if (shift != 0) {
- const size_t valueBitCount = ValueBitCount();
- // Do nothing for empty bitmap
- if (valueBitCount != 0) {
- const size_t eshift = shift / BitsPerChunk;
- const size_t offset = shift % BitsPerChunk;
- const size_t subOffset = BitsPerChunk - offset;
-
- // Don't verify expand result, so l-shift of fixed bitmap will work in the same way as for unsigned integer.
- Mask.ExpandBitSize(valueBitCount + shift);
-
- if (offset == 0) {
- for (size_t i = Mask.GetChunkCapacity() - 1; i >= eshift; --i) {
- Mask.Data[i] = Mask.Data[i - eshift];
- }
- } else {
- for (size_t i = Mask.GetChunkCapacity() - 1; i > eshift; --i)
- Mask.Data[i] = (Mask.Data[i - eshift] << offset) | (Mask.Data[i - eshift - 1] >> subOffset);
- if (eshift < Mask.GetChunkCapacity())
- Mask.Data[eshift] = Mask.Data[0] << offset;
- }
- for (size_t i = 0; i < Min(eshift, Mask.GetChunkCapacity()); ++i)
- Mask.Data[i] = 0;
-
- // Cleanup extra high bits in the storage
- Mask.Sanitize();
- }
- }
- return *this;
- }
-
- TThis& RShift(size_t shift) {
- if (shift != 0) {
- const size_t eshift = shift / BitsPerChunk;
- const size_t offset = shift % BitsPerChunk;
- if (eshift >= Mask.GetChunkCapacity()) {
- Clear();
-
- } else {
- const size_t limit = Mask.GetChunkCapacity() - eshift - 1;
-
- if (offset == 0) {
- for (size_t i = 0; i <= limit; ++i) {
- Mask.Data[i] = Mask.Data[i + eshift];
- }
- } else {
- const size_t subOffset = BitsPerChunk - offset;
- for (size_t i = 0; i < limit; ++i)
- Mask.Data[i] = (Mask.Data[i + eshift] >> offset) | (Mask.Data[i + eshift + 1] << subOffset);
- Mask.Data[limit] = Mask.Data[Mask.GetChunkCapacity() - 1] >> offset;
- }
-
- for (size_t i = limit + 1; i < Mask.GetChunkCapacity(); ++i)
- Mask.Data[i] = 0;
- }
- }
- return *this;
- }
-
- // Applies bitmap at the specified offset using OR operator.
- // This method is optimized combination of Or() and LShift(), which allows reducing memory allocation
- // when combining long dynamic bitmaps.
- TThis& Or(const TThis& bitmap, size_t offset) {
- if (0 == offset)
- return Or(bitmap);
-
- const size_t otherValueBitCount = bitmap.ValueBitCount();
- // Continue only if OR-ed bitmap have non-zero bits
- if (otherValueBitCount) {
- const size_t chunkShift = offset / BitsPerChunk;
- const size_t subShift = offset % BitsPerChunk;
- const size_t subOffset = BitsPerChunk - subShift;
-
- Reserve(otherValueBitCount + offset);
-
- if (subShift == 0) {
- for (size_t i = chunkShift; i < Min(bitmap.Mask.GetChunkCapacity() + chunkShift, Mask.GetChunkCapacity()); ++i) {
- Mask.Data[i] |= bitmap.Mask.Data[i - chunkShift];
- }
- } else {
- Mask.Data[chunkShift] |= bitmap.Mask.Data[0] << subShift;
- size_t i = chunkShift + 1;
- for (; i < Min(bitmap.Mask.GetChunkCapacity() + chunkShift, Mask.GetChunkCapacity()); ++i) {
- Mask.Data[i] |= (bitmap.Mask.Data[i - chunkShift] << subShift) | (bitmap.Mask.Data[i - chunkShift - 1] >> subOffset);
- }
- if (i < Mask.GetChunkCapacity())
- Mask.Data[i] |= bitmap.Mask.Data[i - chunkShift - 1] >> subOffset;
- }
- }
-
- return *this;
- }
-
- bool Equal(const TThis& bitmap) const {
- if (Mask.GetChunkCapacity() > bitmap.Mask.GetChunkCapacity()) {
- for (size_t i = bitmap.Mask.GetChunkCapacity(); i < Mask.GetChunkCapacity(); ++i) {
- if (0 != Mask.Data[i])
- return false;
- }
+ for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i)
+ Mask.Data[i] = ~Mask.Data[i];
+ Mask.Sanitize();
+ return *this;
+ }
+
+ TThis& LShift(size_t shift) {
+ if (shift != 0) {
+ const size_t valueBitCount = ValueBitCount();
+ // Do nothing for empty bitmap
+ if (valueBitCount != 0) {
+ const size_t eshift = shift / BitsPerChunk;
+ const size_t offset = shift % BitsPerChunk;
+ const size_t subOffset = BitsPerChunk - offset;
+
+ // Don't verify expand result, so l-shift of fixed bitmap will work in the same way as for unsigned integer.
+ Mask.ExpandBitSize(valueBitCount + shift);
+
+ if (offset == 0) {
+ for (size_t i = Mask.GetChunkCapacity() - 1; i >= eshift; --i) {
+ Mask.Data[i] = Mask.Data[i - eshift];
+ }
+ } else {
+ for (size_t i = Mask.GetChunkCapacity() - 1; i > eshift; --i)
+ Mask.Data[i] = (Mask.Data[i - eshift] << offset) | (Mask.Data[i - eshift - 1] >> subOffset);
+ if (eshift < Mask.GetChunkCapacity())
+ Mask.Data[eshift] = Mask.Data[0] << offset;
+ }
+ for (size_t i = 0; i < Min(eshift, Mask.GetChunkCapacity()); ++i)
+ Mask.Data[i] = 0;
+
+ // Cleanup extra high bits in the storage
+ Mask.Sanitize();
+ }
+ }
+ return *this;
+ }
+
+ TThis& RShift(size_t shift) {
+ if (shift != 0) {
+ const size_t eshift = shift / BitsPerChunk;
+ const size_t offset = shift % BitsPerChunk;
+ if (eshift >= Mask.GetChunkCapacity()) {
+ Clear();
+
+ } else {
+ const size_t limit = Mask.GetChunkCapacity() - eshift - 1;
+
+ if (offset == 0) {
+ for (size_t i = 0; i <= limit; ++i) {
+ Mask.Data[i] = Mask.Data[i + eshift];
+ }
+ } else {
+ const size_t subOffset = BitsPerChunk - offset;
+ for (size_t i = 0; i < limit; ++i)
+ Mask.Data[i] = (Mask.Data[i + eshift] >> offset) | (Mask.Data[i + eshift + 1] << subOffset);
+ Mask.Data[limit] = Mask.Data[Mask.GetChunkCapacity() - 1] >> offset;
+ }
+
+ for (size_t i = limit + 1; i < Mask.GetChunkCapacity(); ++i)
+ Mask.Data[i] = 0;
+ }
+ }
+ return *this;
+ }
+
+ // Applies bitmap at the specified offset using OR operator.
+ // This method is optimized combination of Or() and LShift(), which allows reducing memory allocation
+ // when combining long dynamic bitmaps.
+ TThis& Or(const TThis& bitmap, size_t offset) {
+ if (0 == offset)
+ return Or(bitmap);
+
+ const size_t otherValueBitCount = bitmap.ValueBitCount();
+ // Continue only if OR-ed bitmap have non-zero bits
+ if (otherValueBitCount) {
+ const size_t chunkShift = offset / BitsPerChunk;
+ const size_t subShift = offset % BitsPerChunk;
+ const size_t subOffset = BitsPerChunk - subShift;
+
+ Reserve(otherValueBitCount + offset);
+
+ if (subShift == 0) {
+ for (size_t i = chunkShift; i < Min(bitmap.Mask.GetChunkCapacity() + chunkShift, Mask.GetChunkCapacity()); ++i) {
+ Mask.Data[i] |= bitmap.Mask.Data[i - chunkShift];
+ }
+ } else {
+ Mask.Data[chunkShift] |= bitmap.Mask.Data[0] << subShift;
+ size_t i = chunkShift + 1;
+ for (; i < Min(bitmap.Mask.GetChunkCapacity() + chunkShift, Mask.GetChunkCapacity()); ++i) {
+ Mask.Data[i] |= (bitmap.Mask.Data[i - chunkShift] << subShift) | (bitmap.Mask.Data[i - chunkShift - 1] >> subOffset);
+ }
+ if (i < Mask.GetChunkCapacity())
+ Mask.Data[i] |= bitmap.Mask.Data[i - chunkShift - 1] >> subOffset;
+ }
+ }
+
+ return *this;
+ }
+
+ bool Equal(const TThis& bitmap) const {
+ if (Mask.GetChunkCapacity() > bitmap.Mask.GetChunkCapacity()) {
+ for (size_t i = bitmap.Mask.GetChunkCapacity(); i < Mask.GetChunkCapacity(); ++i) {
+ if (0 != Mask.Data[i])
+ return false;
+ }
} else if (Mask.GetChunkCapacity() < bitmap.Mask.GetChunkCapacity()) {
- for (size_t i = Mask.GetChunkCapacity(); i < bitmap.Mask.GetChunkCapacity(); ++i) {
- if (0 != bitmap.Mask.Data[i])
- return false;
- }
- }
- size_t size = Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity());
- for (size_t i = 0; i < size; ++i) {
- if (Mask.Data[i] != bitmap.Mask.Data[i])
- return false;
- }
- return true;
- }
-
- template <class T>
+ for (size_t i = Mask.GetChunkCapacity(); i < bitmap.Mask.GetChunkCapacity(); ++i) {
+ if (0 != bitmap.Mask.Data[i])
+ return false;
+ }
+ }
+ size_t size = Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity());
+ for (size_t i = 0; i < size; ++i) {
+ if (Mask.Data[i] != bitmap.Mask.Data[i])
+ return false;
+ }
+ return true;
+ }
+
+ template <class T>
Y_FORCE_INLINE bool Equal(const TBitMapOps<T>& bitmap) const {
- return Equal(TThis(bitmap));
- }
-
- int Compare(const TThis& bitmap) const {
- size_t size = Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity());
- int res = ::memcmp(Mask.Data, bitmap.Mask.Data, size * sizeof(TChunk));
- if (0 != res || Mask.GetChunkCapacity() == bitmap.Mask.GetChunkCapacity())
- return res;
-
- if (Mask.GetChunkCapacity() > bitmap.Mask.GetChunkCapacity()) {
- for (size_t i = bitmap.Mask.GetChunkCapacity(); i < Mask.GetChunkCapacity(); ++i) {
- if (0 != Mask.Data[i])
- return 1;
- }
- } else {
- for (size_t i = Mask.GetChunkCapacity(); i < bitmap.Mask.GetChunkCapacity(); ++i) {
- if (0 != bitmap.Mask.Data[i])
- return -1;
- }
- }
- return 0;
- }
-
- template <class T>
+ return Equal(TThis(bitmap));
+ }
+
+ int Compare(const TThis& bitmap) const {
+ size_t size = Min(Mask.GetChunkCapacity(), bitmap.Mask.GetChunkCapacity());
+ int res = ::memcmp(Mask.Data, bitmap.Mask.Data, size * sizeof(TChunk));
+ if (0 != res || Mask.GetChunkCapacity() == bitmap.Mask.GetChunkCapacity())
+ return res;
+
+ if (Mask.GetChunkCapacity() > bitmap.Mask.GetChunkCapacity()) {
+ for (size_t i = bitmap.Mask.GetChunkCapacity(); i < Mask.GetChunkCapacity(); ++i) {
+ if (0 != Mask.Data[i])
+ return 1;
+ }
+ } else {
+ for (size_t i = Mask.GetChunkCapacity(); i < bitmap.Mask.GetChunkCapacity(); ++i) {
+ if (0 != bitmap.Mask.Data[i])
+ return -1;
+ }
+ }
+ return 0;
+ }
+
+ template <class T>
Y_FORCE_INLINE int Compare(const TBitMapOps<T>& bitmap) const {
- return Compare(TThis(bitmap));
- }
-
- // For backward compatibility
+ return Compare(TThis(bitmap));
+ }
+
+ // For backward compatibility
Y_FORCE_INLINE static int Compare(const TThis& l, const TThis& r) {
- return l.Compare(r);
- }
-
+ return l.Compare(r);
+ }
+
size_t FirstNonZeroBit() const {
- for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i) {
- if (Mask.Data[i]) {
- // CountTrailingZeroBits() expects unsigned types not smaller than unsigned int. So, convert before calling
- return BitsPerChunk * i + CountTrailingZeroBits(TIntType(Mask.Data[i]));
- }
- }
- return Size();
- }
-
- // Returns position of the next non-zero bit, which offset is greater than specified pos
- // Typical loop for iterating bits:
- // for (size_t pos = bits.FirstNonZeroBit(); pos != bits.Size(); pos = bits.NextNonZeroBit(pos)) {
- // ...
- // }
+ for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i) {
+ if (Mask.Data[i]) {
+ // CountTrailingZeroBits() expects unsigned types not smaller than unsigned int. So, convert before calling
+ return BitsPerChunk * i + CountTrailingZeroBits(TIntType(Mask.Data[i]));
+ }
+ }
+ return Size();
+ }
+
+ // Returns position of the next non-zero bit, which offset is greater than specified pos
+ // Typical loop for iterating bits:
+ // for (size_t pos = bits.FirstNonZeroBit(); pos != bits.Size(); pos = bits.NextNonZeroBit(pos)) {
+ // ...
+ // }
// See Y_FOR_EACH_BIT macro definition at the bottom
- size_t NextNonZeroBit(size_t pos) const {
+ size_t NextNonZeroBit(size_t pos) const {
size_t i = (pos + 1) >> DivCount;
- if (i < Mask.GetChunkCapacity()) {
- const size_t offset = (pos + 1) & ModMask;
- // Process the current chunk
- if (offset) {
- // Zero already iterated trailing bits using mask
- const TChunk val = Mask.Data[i] & ((~TChunk(0)) << offset);
- if (val) {
- return BitsPerChunk * i + CountTrailingZeroBits(TIntType(val));
- }
- // Continue with other chunks
- ++i;
- }
-
- for (; i < Mask.GetChunkCapacity(); ++i) {
- if (Mask.Data[i]) {
- return BitsPerChunk * i + CountTrailingZeroBits(TIntType(Mask.Data[i]));
+ if (i < Mask.GetChunkCapacity()) {
+ const size_t offset = (pos + 1) & ModMask;
+ // Process the current chunk
+ if (offset) {
+ // Zero already iterated trailing bits using mask
+ const TChunk val = Mask.Data[i] & ((~TChunk(0)) << offset);
+ if (val) {
+ return BitsPerChunk * i + CountTrailingZeroBits(TIntType(val));
}
+ // Continue with other chunks
+ ++i;
}
+
+ for (; i < Mask.GetChunkCapacity(); ++i) {
+ if (Mask.Data[i]) {
+ return BitsPerChunk * i + CountTrailingZeroBits(TIntType(Mask.Data[i]));
+ }
+ }
}
- return Size();
+ return Size();
}
Y_FORCE_INLINE size_t Count() const {
size_t count = 0;
- for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i)
+ for (size_t i = 0; i < Mask.GetChunkCapacity(); ++i)
count += ::NBitMapPrivate::CountBitsPrivate(Mask.Data[i]);
return count;
}
@@ -983,7 +983,7 @@ public:
if (chunkCount < Mask.GetChunkCapacity()) {
::memset(Mask.Data + chunkCount, 0, (Mask.GetChunkCapacity() - chunkCount) * sizeof(TChunk));
}
- Mask.Sanitize();
+ Mask.Sanitize();
}
inline size_t Hash() const {
@@ -1009,58 +1009,58 @@ public:
return Mask.GetChunkCapacity();
}
};
-
-template <class X, class Y>
+
+template <class X, class Y>
inline TBitMapOps<X> operator&(const TBitMapOps<X>& x, const TBitMapOps<Y>& y) {
- return TBitMapOps<X>(x).And(y);
-}
-
-template <class X>
+ return TBitMapOps<X>(x).And(y);
+}
+
+template <class X>
inline TBitMapOps<X> operator&(const TBitMapOps<X>& x, const typename TBitMapOps<X>::TChunk& y) {
- return TBitMapOps<X>(x).And(y);
-}
-
-template <class X>
+ return TBitMapOps<X>(x).And(y);
+}
+
+template <class X>
inline TBitMapOps<X> operator&(const typename TBitMapOps<X>::TChunk& x, const TBitMapOps<X>& y) {
- return TBitMapOps<X>(x).And(y);
-}
-
-template <class X, class Y>
+ return TBitMapOps<X>(x).And(y);
+}
+
+template <class X, class Y>
inline TBitMapOps<X> operator|(const TBitMapOps<X>& x, const TBitMapOps<Y>& y) {
- return TBitMapOps<X>(x).Or(y);
-}
-
-template <class X>
+ return TBitMapOps<X>(x).Or(y);
+}
+
+template <class X>
inline TBitMapOps<X> operator|(const TBitMapOps<X>& x, const typename TBitMapOps<X>::TChunk& y) {
- return TBitMapOps<X>(x).Or(y);
-}
-
-template <class X>
+ return TBitMapOps<X>(x).Or(y);
+}
+
+template <class X>
inline TBitMapOps<X> operator|(const typename TBitMapOps<X>::TChunk& x, const TBitMapOps<X>& y) {
- return TBitMapOps<X>(x).Or(y);
-}
-
-template <class X, class Y>
+ return TBitMapOps<X>(x).Or(y);
+}
+
+template <class X, class Y>
inline TBitMapOps<X> operator^(const TBitMapOps<X>& x, const TBitMapOps<Y>& y) {
- return TBitMapOps<X>(x).Xor(y);
-}
-
-template <class X>
+ return TBitMapOps<X>(x).Xor(y);
+}
+
+template <class X>
inline TBitMapOps<X> operator^(const TBitMapOps<X>& x, const typename TBitMapOps<X>::TChunk& y) {
- return TBitMapOps<X>(x).Xor(y);
-}
-
-template <class X>
+ return TBitMapOps<X>(x).Xor(y);
+}
+
+template <class X>
inline TBitMapOps<X> operator^(const typename TBitMapOps<X>::TChunk& x, const TBitMapOps<X>& y) {
- return TBitMapOps<X>(x).Xor(y);
-}
-
+ return TBitMapOps<X>(x).Xor(y);
+}
+
template <class X, class Y>
inline TBitMapOps<X> operator-(const TBitMapOps<X>& x, const TBitMapOps<Y>& y) {
return TBitMapOps<X>(x).SetDifference(y);
}
-template <class X>
+template <class X>
inline TBitMapOps<X> operator-(const TBitMapOps<X>& x, const typename TBitMapOps<X>::TChunk& y) {
return TBitMapOps<X>(x).SetDifference(y);
}
@@ -1072,38 +1072,38 @@ inline TBitMapOps<X> operator-(const typename TBitMapOps<X>::TChunk& x, const TB
template <class X>
inline TBitMapOps<X> operator~(const TBitMapOps<X>& x) {
- return TBitMapOps<X>(x).Flip();
-}
-
-/////////////////// Specialization ///////////////////////////
-
+ return TBitMapOps<X>(x).Flip();
+}
+
+/////////////////// Specialization ///////////////////////////
+
template <size_t BitCount, typename TChunkType /*= ui64*/>
class TBitMap: public TBitMapOps<TFixedBitMapTraits<BitCount, TChunkType>> {
-private:
+private:
using TBase = TBitMapOps<TFixedBitMapTraits<BitCount, TChunkType>>;
-public:
- TBitMap()
- : TBase()
- {
- }
-
- TBitMap(typename TBase::TChunk val)
- : TBase(val)
- {
- }
-
+public:
+ TBitMap()
+ : TBase()
+ {
+ }
+
+ TBitMap(typename TBase::TChunk val)
+ : TBase(val)
+ {
+ }
+
TBitMap(const TBitMap<BitCount, TChunkType>&) = default;
-
- template <class T>
- TBitMap(const TBitMapOps<T>& bitmap)
- : TBase(bitmap)
- {
- }
-};
-
+
+ template <class T>
+ TBitMap(const TBitMapOps<T>& bitmap)
+ : TBase(bitmap)
+ {
+ }
+};
+
using TDynBitMap = TBitMapOps<TDynamicBitMapTraits<ui64>>;
-
+
#define Y_FOR_EACH_BIT(var, bitmap) for (size_t var = (bitmap).FirstNonZeroBit(); var != (bitmap).Size(); var = (bitmap).NextNonZeroBit(var))
template <typename TTraits>
diff --git a/util/generic/bitmap_ut.cpp b/util/generic/bitmap_ut.cpp
index 087d34a8dc..67156303a0 100644
--- a/util/generic/bitmap_ut.cpp
+++ b/util/generic/bitmap_ut.cpp
@@ -3,10 +3,10 @@
#include <library/cpp/testing/unittest/registar.h>
#define INIT_BITMAP(bitmap, bits) \
- for (size_t i = 0; i < sizeof(bits) / sizeof(size_t); ++i) { \
+ for (size_t i = 0; i < sizeof(bits) / sizeof(size_t); ++i) { \
bitmap.Set(bits[i]); \
- }
-
+ }
+
#define CHECK_BITMAP(bitmap, bits) \
{ \
size_t cur = 0, end = sizeof(bits) / sizeof(size_t); \
@@ -15,11 +15,11 @@
UNIT_ASSERT_EQUAL_C(bitmap.Get(i), true, "pos=" << i); \
++cur; \
} else { \
- UNIT_ASSERT_EQUAL_C(bitmap.Get(i), false, "pos=" << i); \
+ UNIT_ASSERT_EQUAL_C(bitmap.Get(i), false, "pos=" << i); \
} \
} \
- }
-
+ }
+
#define CHECK_BITMAP_WITH_TAIL(bitmap, bits) \
{ \
size_t cur = 0, end = sizeof(bits) / sizeof(size_t); \
@@ -29,14 +29,14 @@
UNIT_ASSERT_EQUAL_C(bitmap.Get(i), true, "pos=" << i); \
++cur; \
} else { \
- UNIT_ASSERT_EQUAL_C(bitmap.Get(i), false, "pos=" << i); \
+ UNIT_ASSERT_EQUAL_C(bitmap.Get(i), false, "pos=" << i); \
} \
} else { \
UNIT_ASSERT_EQUAL_C(bitmap.Get(i), true, "pos=" << i); \
} \
} \
- }
-
+ }
+
Y_UNIT_TEST_SUITE(TBitMapTest) {
Y_UNIT_TEST(TestBitMap) {
TBitMap<101> bitmap;
@@ -45,47 +45,47 @@ Y_UNIT_TEST_SUITE(TBitMapTest) {
UNIT_ASSERT_EQUAL(bitmap.Count(), 0);
UNIT_ASSERT_EQUAL(bitmap.FirstNonZeroBit(), 101);
- size_t initBits[] = {0, 50, 100, 45};
- INIT_BITMAP(bitmap, initBits);
+ size_t initBits[] = {0, 50, 100, 45};
+ INIT_BITMAP(bitmap, initBits);
bitmap.Reset(45);
UNIT_ASSERT_EQUAL(bitmap.FirstNonZeroBit(), 0);
- size_t setBits[] = {0, 50, 100};
- CHECK_BITMAP(bitmap, setBits);
+ size_t setBits[] = {0, 50, 100};
+ CHECK_BITMAP(bitmap, setBits);
for (size_t i = 0; i < bitmap.Size(); ++i) {
- UNIT_ASSERT_EQUAL(bitmap.Get(i), bitmap.Test(i));
+ UNIT_ASSERT_EQUAL(bitmap.Get(i), bitmap.Test(i));
}
- UNIT_ASSERT_EQUAL(bitmap.Count(), 3);
-
- bitmap.Reset(0);
-
- UNIT_ASSERT_EQUAL(bitmap.FirstNonZeroBit(), 50);
-
- bitmap.Clear();
-
- UNIT_ASSERT_EQUAL(bitmap.Count(), 0);
- UNIT_ASSERT_EQUAL(bitmap.Empty(), true);
- }
-
+ UNIT_ASSERT_EQUAL(bitmap.Count(), 3);
+
+ bitmap.Reset(0);
+
+ UNIT_ASSERT_EQUAL(bitmap.FirstNonZeroBit(), 50);
+
+ bitmap.Clear();
+
+ UNIT_ASSERT_EQUAL(bitmap.Count(), 0);
+ UNIT_ASSERT_EQUAL(bitmap.Empty(), true);
+ }
+
Y_UNIT_TEST(TestDynBitMap) {
- TDynBitMap bitmap;
-
- UNIT_ASSERT_EQUAL(bitmap.Size(), 64); // Initial capacity
- UNIT_ASSERT_EQUAL(bitmap.Count(), 0);
- UNIT_ASSERT_EQUAL(bitmap.FirstNonZeroBit(), 64);
-
- size_t initBits[] = {0, 50, 100, 45};
- INIT_BITMAP(bitmap, initBits);
- bitmap.Reset(45);
-
- UNIT_ASSERT_EQUAL(bitmap.Size(), 128);
-
- UNIT_ASSERT_EQUAL(bitmap.FirstNonZeroBit(), 0);
- size_t setBits[] = {0, 50, 100};
- CHECK_BITMAP(bitmap, setBits);
-
+ TDynBitMap bitmap;
+
+ UNIT_ASSERT_EQUAL(bitmap.Size(), 64); // Initial capacity
+ UNIT_ASSERT_EQUAL(bitmap.Count(), 0);
+ UNIT_ASSERT_EQUAL(bitmap.FirstNonZeroBit(), 64);
+
+ size_t initBits[] = {0, 50, 100, 45};
+ INIT_BITMAP(bitmap, initBits);
+ bitmap.Reset(45);
+
+ UNIT_ASSERT_EQUAL(bitmap.Size(), 128);
+
+ UNIT_ASSERT_EQUAL(bitmap.FirstNonZeroBit(), 0);
+ size_t setBits[] = {0, 50, 100};
+ CHECK_BITMAP(bitmap, setBits);
+
for (size_t i = 0; i < bitmap.Size(); ++i) {
UNIT_ASSERT_EQUAL(bitmap.Get(i), bitmap.Test(i));
}
@@ -99,171 +99,171 @@ Y_UNIT_TEST_SUITE(TBitMapTest) {
bitmap.Clear();
UNIT_ASSERT_EQUAL(bitmap.Count(), 0);
- UNIT_ASSERT_EQUAL(bitmap.Empty(), true);
+ UNIT_ASSERT_EQUAL(bitmap.Empty(), true);
}
- template <class TBitMapImpl>
- void TestAndImpl() {
- TBitMapImpl bitmap1;
- TBitMapImpl bitmap2;
-
- size_t initBits1[] = {10, 20, 50, 100};
- size_t initBits2[] = {10, 11, 22, 50};
-
- INIT_BITMAP(bitmap1, initBits1);
- INIT_BITMAP(bitmap2, initBits2);
-
+ template <class TBitMapImpl>
+ void TestAndImpl() {
+ TBitMapImpl bitmap1;
+ TBitMapImpl bitmap2;
+
+ size_t initBits1[] = {10, 20, 50, 100};
+ size_t initBits2[] = {10, 11, 22, 50};
+
+ INIT_BITMAP(bitmap1, initBits1);
+ INIT_BITMAP(bitmap2, initBits2);
+
bitmap1.And(bitmap2);
UNIT_ASSERT_EQUAL(bitmap1.Count(), 2);
UNIT_ASSERT_EQUAL(bitmap2.Count(), 4);
- size_t resBits[] = {10, 50};
+ size_t resBits[] = {10, 50};
- CHECK_BITMAP(bitmap1, resBits);
- CHECK_BITMAP(bitmap2, initBits2);
+ CHECK_BITMAP(bitmap1, resBits);
+ CHECK_BITMAP(bitmap2, initBits2);
}
Y_UNIT_TEST(TestAndFixed) {
TestAndImpl<TBitMap<101>>();
- }
+ }
Y_UNIT_TEST(TestAndDyn) {
- TestAndImpl<TDynBitMap>();
- }
-
- template <class TBitMapImpl>
- void TestOrImpl() {
- TBitMapImpl bitmap1;
- TBitMapImpl bitmap2;
-
- size_t initBits1[] = {0, 10, 11, 76};
- size_t initBits2[] = {1, 11, 22, 50};
-
- INIT_BITMAP(bitmap1, initBits1);
- INIT_BITMAP(bitmap2, initBits2);
-
+ TestAndImpl<TDynBitMap>();
+ }
+
+ template <class TBitMapImpl>
+ void TestOrImpl() {
+ TBitMapImpl bitmap1;
+ TBitMapImpl bitmap2;
+
+ size_t initBits1[] = {0, 10, 11, 76};
+ size_t initBits2[] = {1, 11, 22, 50};
+
+ INIT_BITMAP(bitmap1, initBits1);
+ INIT_BITMAP(bitmap2, initBits2);
+
bitmap1.Or(bitmap2);
UNIT_ASSERT_EQUAL(bitmap1.Count(), 7);
UNIT_ASSERT_EQUAL(bitmap2.Count(), 4);
- size_t resBits[] = {0, 1, 10, 11, 22, 50, 76};
-
- CHECK_BITMAP(bitmap1, resBits);
- CHECK_BITMAP(bitmap2, initBits2);
-
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits1);
-
- UNIT_ASSERT_EQUAL(bitmap1 | (bitmap2 << 3), TBitMapImpl(bitmap1).Or(bitmap2, 3));
- UNIT_ASSERT_EQUAL(bitmap1 | (bitmap2 << 64), TBitMapImpl(bitmap1).Or(bitmap2, 64));
- UNIT_ASSERT_EQUAL(bitmap1 | (bitmap2 << 66), TBitMapImpl(bitmap1).Or(bitmap2, 66));
-
- UNIT_ASSERT_EQUAL(bitmap2 | (bitmap1 << 3), TBitMapImpl(bitmap2).Or(bitmap1, 3));
- UNIT_ASSERT_EQUAL(bitmap2 | (bitmap1 << 64), TBitMapImpl(bitmap2).Or(bitmap1, 64));
- UNIT_ASSERT_EQUAL(bitmap2 | (bitmap1 << 66), TBitMapImpl(bitmap2).Or(bitmap1, 66));
- }
-
+ size_t resBits[] = {0, 1, 10, 11, 22, 50, 76};
+
+ CHECK_BITMAP(bitmap1, resBits);
+ CHECK_BITMAP(bitmap2, initBits2);
+
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits1);
+
+ UNIT_ASSERT_EQUAL(bitmap1 | (bitmap2 << 3), TBitMapImpl(bitmap1).Or(bitmap2, 3));
+ UNIT_ASSERT_EQUAL(bitmap1 | (bitmap2 << 64), TBitMapImpl(bitmap1).Or(bitmap2, 64));
+ UNIT_ASSERT_EQUAL(bitmap1 | (bitmap2 << 66), TBitMapImpl(bitmap1).Or(bitmap2, 66));
+
+ UNIT_ASSERT_EQUAL(bitmap2 | (bitmap1 << 3), TBitMapImpl(bitmap2).Or(bitmap1, 3));
+ UNIT_ASSERT_EQUAL(bitmap2 | (bitmap1 << 64), TBitMapImpl(bitmap2).Or(bitmap1, 64));
+ UNIT_ASSERT_EQUAL(bitmap2 | (bitmap1 << 66), TBitMapImpl(bitmap2).Or(bitmap1, 66));
+ }
+
Y_UNIT_TEST(TestOrFixed) {
TestOrImpl<TBitMap<145>>();
- }
-
+ }
+
Y_UNIT_TEST(TestOrDyn) {
- TestOrImpl<TDynBitMap>();
- }
-
+ TestOrImpl<TDynBitMap>();
+ }
+
Y_UNIT_TEST(TestCopy) {
- TBitMap<101> bitmap1;
- size_t initBits[] = {0, 10, 11, 76, 100};
-
- INIT_BITMAP(bitmap1, initBits);
-
- TDynBitMap bitmap2(bitmap1);
- CHECK_BITMAP(bitmap2, initBits);
-
- TBitMap<101> bitmap3(bitmap1);
- CHECK_BITMAP(bitmap3, initBits);
-
- TBitMap<127> bitmap4(bitmap1);
- CHECK_BITMAP(bitmap4, initBits);
-
- TDynBitMap bitmap5;
- bitmap5 = bitmap1;
- CHECK_BITMAP(bitmap5, initBits);
-
- TBitMap<101> bitmap6;
- bitmap6 = bitmap1;
- CHECK_BITMAP(bitmap6, initBits);
-
- TBitMap<127> bitmap7;
- bitmap7 = bitmap1;
- CHECK_BITMAP(bitmap7, initBits);
-
- TBitMap<101> bitmap8;
- DoSwap(bitmap8, bitmap6);
- CHECK_BITMAP(bitmap8, initBits);
- UNIT_ASSERT_EQUAL(bitmap6.Empty(), true);
-
- TDynBitMap bitmap9;
- DoSwap(bitmap9, bitmap5);
- CHECK_BITMAP(bitmap9, initBits);
- UNIT_ASSERT_EQUAL(bitmap5.Empty(), true);
-
- // 64->32
- TBitMap<160, ui32> bitmap10(bitmap1);
- CHECK_BITMAP(bitmap10, initBits);
-
- // 64->16
- TBitMap<160, ui16> bitmap11(bitmap1);
- CHECK_BITMAP(bitmap11, initBits);
-
- // 64->8
- TBitMap<160, ui8> bitmap12(bitmap1);
- CHECK_BITMAP(bitmap12, initBits);
-
- // 32->16
- TBitMap<160, ui16> bitmap13(bitmap10);
- CHECK_BITMAP(bitmap13, initBits);
-
- // 32->64
- TBitMap<160, ui64> bitmap14(bitmap10);
- CHECK_BITMAP(bitmap14, initBits);
-
- // 16->64
- TBitMap<160, ui64> bitmap15(bitmap11);
- CHECK_BITMAP(bitmap15, initBits);
-
- // 8->64
- TBitMap<160, ui64> bitmap16(bitmap12);
- CHECK_BITMAP(bitmap16, initBits);
- }
-
+ TBitMap<101> bitmap1;
+ size_t initBits[] = {0, 10, 11, 76, 100};
+
+ INIT_BITMAP(bitmap1, initBits);
+
+ TDynBitMap bitmap2(bitmap1);
+ CHECK_BITMAP(bitmap2, initBits);
+
+ TBitMap<101> bitmap3(bitmap1);
+ CHECK_BITMAP(bitmap3, initBits);
+
+ TBitMap<127> bitmap4(bitmap1);
+ CHECK_BITMAP(bitmap4, initBits);
+
+ TDynBitMap bitmap5;
+ bitmap5 = bitmap1;
+ CHECK_BITMAP(bitmap5, initBits);
+
+ TBitMap<101> bitmap6;
+ bitmap6 = bitmap1;
+ CHECK_BITMAP(bitmap6, initBits);
+
+ TBitMap<127> bitmap7;
+ bitmap7 = bitmap1;
+ CHECK_BITMAP(bitmap7, initBits);
+
+ TBitMap<101> bitmap8;
+ DoSwap(bitmap8, bitmap6);
+ CHECK_BITMAP(bitmap8, initBits);
+ UNIT_ASSERT_EQUAL(bitmap6.Empty(), true);
+
+ TDynBitMap bitmap9;
+ DoSwap(bitmap9, bitmap5);
+ CHECK_BITMAP(bitmap9, initBits);
+ UNIT_ASSERT_EQUAL(bitmap5.Empty(), true);
+
+ // 64->32
+ TBitMap<160, ui32> bitmap10(bitmap1);
+ CHECK_BITMAP(bitmap10, initBits);
+
+ // 64->16
+ TBitMap<160, ui16> bitmap11(bitmap1);
+ CHECK_BITMAP(bitmap11, initBits);
+
+ // 64->8
+ TBitMap<160, ui8> bitmap12(bitmap1);
+ CHECK_BITMAP(bitmap12, initBits);
+
+ // 32->16
+ TBitMap<160, ui16> bitmap13(bitmap10);
+ CHECK_BITMAP(bitmap13, initBits);
+
+ // 32->64
+ TBitMap<160, ui64> bitmap14(bitmap10);
+ CHECK_BITMAP(bitmap14, initBits);
+
+ // 16->64
+ TBitMap<160, ui64> bitmap15(bitmap11);
+ CHECK_BITMAP(bitmap15, initBits);
+
+ // 8->64
+ TBitMap<160, ui64> bitmap16(bitmap12);
+ CHECK_BITMAP(bitmap16, initBits);
+ }
+
Y_UNIT_TEST(TestOps) {
- TBitMap<16> bitmap1;
- TBitMap<12> bitmap2;
- size_t initBits1[] = {0, 3, 7, 11};
- size_t initBits2[] = {1, 3, 6, 7, 11};
- INIT_BITMAP(bitmap1, initBits1);
- INIT_BITMAP(bitmap2, initBits2);
-
- bitmap1.Or(3).And(bitmap2).Flip();
-
- size_t resBits[] = {0, 2, 4, 5, 6, 8, 9, 10, 12};
- CHECK_BITMAP_WITH_TAIL(bitmap1, resBits);
-
- TDynBitMap bitmap3;
- INIT_BITMAP(bitmap3, initBits1);
-
- bitmap3.Or(3).And(bitmap2).Flip();
-
- CHECK_BITMAP_WITH_TAIL(bitmap3, resBits);
-
- bitmap3.Clear();
- INIT_BITMAP(bitmap3, initBits1);
-
+ TBitMap<16> bitmap1;
+ TBitMap<12> bitmap2;
+ size_t initBits1[] = {0, 3, 7, 11};
+ size_t initBits2[] = {1, 3, 6, 7, 11};
+ INIT_BITMAP(bitmap1, initBits1);
+ INIT_BITMAP(bitmap2, initBits2);
+
+ bitmap1.Or(3).And(bitmap2).Flip();
+
+ size_t resBits[] = {0, 2, 4, 5, 6, 8, 9, 10, 12};
+ CHECK_BITMAP_WITH_TAIL(bitmap1, resBits);
+
+ TDynBitMap bitmap3;
+ INIT_BITMAP(bitmap3, initBits1);
+
+ bitmap3.Or(3).And(bitmap2).Flip();
+
+ CHECK_BITMAP_WITH_TAIL(bitmap3, resBits);
+
+ bitmap3.Clear();
+ INIT_BITMAP(bitmap3, initBits1);
+
TDynBitMap bitmap4 = ~((bitmap3 | 3) & bitmap2);
- CHECK_BITMAP_WITH_TAIL(bitmap4, resBits);
+ CHECK_BITMAP_WITH_TAIL(bitmap4, resBits);
TBitMap<128, ui32> expmap;
expmap.Set(47);
@@ -284,196 +284,196 @@ Y_UNIT_TEST_SUITE(TBitMapTest) {
UNIT_ASSERT_EQUAL(tst2, (1 << 14));
expmap.Export(33, tst3);
UNIT_ASSERT_EQUAL(tst3, (1 << 14));
- }
-
+ }
+
Y_UNIT_TEST(TestShiftFixed) {
- size_t initBits[] = {0, 3, 7, 11};
-
- TBitMap<128> bitmap1;
-
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 62;
- size_t resBits1[] = {62, 65, 69, 73};
- CHECK_BITMAP(bitmap1, resBits1);
- bitmap1 >>= 62;
- CHECK_BITMAP(bitmap1, initBits);
-
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 120;
- size_t resBits2[] = {120, 123, 127};
- CHECK_BITMAP(bitmap1, resBits2);
- bitmap1 >>= 120;
- size_t resBits3[] = {0, 3, 7};
- CHECK_BITMAP(bitmap1, resBits3);
-
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 128;
- UNIT_ASSERT_EQUAL(bitmap1.Empty(), true);
-
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 120;
- bitmap1 >>= 128;
- UNIT_ASSERT_EQUAL(bitmap1.Empty(), true);
-
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 140;
- UNIT_ASSERT_EQUAL(bitmap1.Empty(), true);
-
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 62;
- bitmap1 >>= 140;
- UNIT_ASSERT_EQUAL(bitmap1.Empty(), true);
- }
-
+ size_t initBits[] = {0, 3, 7, 11};
+
+ TBitMap<128> bitmap1;
+
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 62;
+ size_t resBits1[] = {62, 65, 69, 73};
+ CHECK_BITMAP(bitmap1, resBits1);
+ bitmap1 >>= 62;
+ CHECK_BITMAP(bitmap1, initBits);
+
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 120;
+ size_t resBits2[] = {120, 123, 127};
+ CHECK_BITMAP(bitmap1, resBits2);
+ bitmap1 >>= 120;
+ size_t resBits3[] = {0, 3, 7};
+ CHECK_BITMAP(bitmap1, resBits3);
+
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 128;
+ UNIT_ASSERT_EQUAL(bitmap1.Empty(), true);
+
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 120;
+ bitmap1 >>= 128;
+ UNIT_ASSERT_EQUAL(bitmap1.Empty(), true);
+
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 140;
+ UNIT_ASSERT_EQUAL(bitmap1.Empty(), true);
+
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 62;
+ bitmap1 >>= 140;
+ UNIT_ASSERT_EQUAL(bitmap1.Empty(), true);
+ }
+
Y_UNIT_TEST(TestShiftDyn) {
- size_t initBits[] = {0, 3, 7, 11};
-
- TDynBitMap bitmap1;
-
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 62;
- size_t resBits1[] = {62, 65, 69, 73};
- CHECK_BITMAP(bitmap1, resBits1);
- bitmap1 >>= 62;
- CHECK_BITMAP(bitmap1, initBits);
-
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 120;
- size_t resBits2[] = {120, 123, 127, 131};
- CHECK_BITMAP(bitmap1, resBits2);
- bitmap1 >>= 120;
- CHECK_BITMAP(bitmap1, initBits);
-
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 128;
- size_t resBits3[] = {128, 131, 135, 139};
- CHECK_BITMAP(bitmap1, resBits3);
-
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 120;
- bitmap1 >>= 128;
- size_t resBits4[] = {3};
- CHECK_BITMAP(bitmap1, resBits4);
-
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits);
- bitmap1 <<= 62;
- bitmap1 >>= 140;
- UNIT_ASSERT_EQUAL(bitmap1.Empty(), true);
- }
-
- // Test that we don't expand bitmap in LShift when high-order bits are zero
+ size_t initBits[] = {0, 3, 7, 11};
+
+ TDynBitMap bitmap1;
+
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 62;
+ size_t resBits1[] = {62, 65, 69, 73};
+ CHECK_BITMAP(bitmap1, resBits1);
+ bitmap1 >>= 62;
+ CHECK_BITMAP(bitmap1, initBits);
+
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 120;
+ size_t resBits2[] = {120, 123, 127, 131};
+ CHECK_BITMAP(bitmap1, resBits2);
+ bitmap1 >>= 120;
+ CHECK_BITMAP(bitmap1, initBits);
+
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 128;
+ size_t resBits3[] = {128, 131, 135, 139};
+ CHECK_BITMAP(bitmap1, resBits3);
+
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 120;
+ bitmap1 >>= 128;
+ size_t resBits4[] = {3};
+ CHECK_BITMAP(bitmap1, resBits4);
+
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits);
+ bitmap1 <<= 62;
+ bitmap1 >>= 140;
+ UNIT_ASSERT_EQUAL(bitmap1.Empty(), true);
+ }
+
+ // Test that we don't expand bitmap in LShift when high-order bits are zero
Y_UNIT_TEST(TestShiftExpansion) {
- UNIT_ASSERT_EQUAL(TDynBitMap().LShift(1).Size(), 64);
- UNIT_ASSERT_EQUAL(TDynBitMap().LShift(65).Size(), 64);
- UNIT_ASSERT_EQUAL(TDynBitMap().LShift(128).Size(), 64);
-
- TDynBitMap bitmap;
- bitmap.Set(62).LShift(1);
- UNIT_ASSERT_EQUAL(bitmap, TDynBitMap().Set(63));
- UNIT_ASSERT_EQUAL(bitmap.Size(), 64);
-
- // Expand explicitly
- bitmap.Set(65);
- UNIT_ASSERT_EQUAL(bitmap.Size(), 128);
-
- bitmap.Clear().Set(0).LShift(1);
- UNIT_ASSERT_EQUAL(bitmap, TDynBitMap().Set(1));
- UNIT_ASSERT_EQUAL(bitmap.Size(), 128);
-
- bitmap.Clear().Set(63).LShift(1);
- UNIT_ASSERT_EQUAL(bitmap, TDynBitMap().Set(64));
- UNIT_ASSERT_EQUAL(bitmap.Size(), 128);
-
- bitmap.Clear().Set(63).LShift(64);
- UNIT_ASSERT_EQUAL(bitmap, TDynBitMap().Set(127));
- UNIT_ASSERT_EQUAL(bitmap.Size(), 128);
-
- bitmap.Clear().Set(62).LShift(129);
- UNIT_ASSERT_EQUAL(bitmap, TDynBitMap().Set(191));
- UNIT_ASSERT_EQUAL(bitmap.Size(), 256);
- }
-
+ UNIT_ASSERT_EQUAL(TDynBitMap().LShift(1).Size(), 64);
+ UNIT_ASSERT_EQUAL(TDynBitMap().LShift(65).Size(), 64);
+ UNIT_ASSERT_EQUAL(TDynBitMap().LShift(128).Size(), 64);
+
+ TDynBitMap bitmap;
+ bitmap.Set(62).LShift(1);
+ UNIT_ASSERT_EQUAL(bitmap, TDynBitMap().Set(63));
+ UNIT_ASSERT_EQUAL(bitmap.Size(), 64);
+
+ // Expand explicitly
+ bitmap.Set(65);
+ UNIT_ASSERT_EQUAL(bitmap.Size(), 128);
+
+ bitmap.Clear().Set(0).LShift(1);
+ UNIT_ASSERT_EQUAL(bitmap, TDynBitMap().Set(1));
+ UNIT_ASSERT_EQUAL(bitmap.Size(), 128);
+
+ bitmap.Clear().Set(63).LShift(1);
+ UNIT_ASSERT_EQUAL(bitmap, TDynBitMap().Set(64));
+ UNIT_ASSERT_EQUAL(bitmap.Size(), 128);
+
+ bitmap.Clear().Set(63).LShift(64);
+ UNIT_ASSERT_EQUAL(bitmap, TDynBitMap().Set(127));
+ UNIT_ASSERT_EQUAL(bitmap.Size(), 128);
+
+ bitmap.Clear().Set(62).LShift(129);
+ UNIT_ASSERT_EQUAL(bitmap, TDynBitMap().Set(191));
+ UNIT_ASSERT_EQUAL(bitmap.Size(), 256);
+ }
+
Y_UNIT_TEST(TestFixedSanity) {
- // test extra-bit cleanup
- UNIT_ASSERT_EQUAL(TBitMap<33>().Set(0).LShift(34).RShift(34).Empty(), true);
- UNIT_ASSERT_EQUAL(TBitMap<88>().Set(0).Set(1).Set(2).LShift(90).RShift(90).Empty(), true);
- UNIT_ASSERT_EQUAL(TBitMap<88>().Flip().RShift(88).Empty(), true);
- UNIT_ASSERT_EQUAL(TBitMap<64>().Flip().LShift(2).RShift(2).Count(), 62);
- UNIT_ASSERT_EQUAL(TBitMap<67>().Flip().LShift(2).RShift(2).Count(), 65);
- UNIT_ASSERT_EQUAL(TBitMap<128>().Flip().LShift(2).RShift(2).Count(), 126);
- UNIT_ASSERT_EQUAL(TBitMap<130>().Flip().LShift(2).RShift(2).Count(), 128);
- UNIT_ASSERT_EQUAL(TBitMap<130>(TDynBitMap().Set(131)).Empty(), true);
- UNIT_ASSERT_EQUAL(TBitMap<33>().Or(TBitMap<40>().Set(39)).Empty(), true);
- UNIT_ASSERT_EQUAL(TBitMap<33>().Xor(TBitMap<40>().Set(39)).Empty(), true);
- }
-
+ // test extra-bit cleanup
+ UNIT_ASSERT_EQUAL(TBitMap<33>().Set(0).LShift(34).RShift(34).Empty(), true);
+ UNIT_ASSERT_EQUAL(TBitMap<88>().Set(0).Set(1).Set(2).LShift(90).RShift(90).Empty(), true);
+ UNIT_ASSERT_EQUAL(TBitMap<88>().Flip().RShift(88).Empty(), true);
+ UNIT_ASSERT_EQUAL(TBitMap<64>().Flip().LShift(2).RShift(2).Count(), 62);
+ UNIT_ASSERT_EQUAL(TBitMap<67>().Flip().LShift(2).RShift(2).Count(), 65);
+ UNIT_ASSERT_EQUAL(TBitMap<128>().Flip().LShift(2).RShift(2).Count(), 126);
+ UNIT_ASSERT_EQUAL(TBitMap<130>().Flip().LShift(2).RShift(2).Count(), 128);
+ UNIT_ASSERT_EQUAL(TBitMap<130>(TDynBitMap().Set(131)).Empty(), true);
+ UNIT_ASSERT_EQUAL(TBitMap<33>().Or(TBitMap<40>().Set(39)).Empty(), true);
+ UNIT_ASSERT_EQUAL(TBitMap<33>().Xor(TBitMap<40>().Set(39)).Empty(), true);
+ }
+
Y_UNIT_TEST(TestIterate) {
- TDynBitMap bitmap1;
- TDynBitMap bitmap2;
-
- size_t initBits1[] = {0, 3, 7, 8, 11, 33, 34, 35, 36, 62, 63, 100, 127};
- INIT_BITMAP(bitmap1, initBits1);
- for (size_t i = bitmap1.FirstNonZeroBit(); i != bitmap1.Size(); i = bitmap1.NextNonZeroBit(i)) {
- bitmap2.Set(i);
- }
- CHECK_BITMAP(bitmap2, initBits1);
- UNIT_ASSERT_EQUAL(bitmap1, bitmap2);
-
- size_t initBits2[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 33, 34, 35, 36, 62};
- bitmap1.Clear();
- bitmap2.Clear();
- INIT_BITMAP(bitmap1, initBits2);
- for (size_t i = bitmap1.FirstNonZeroBit(); i != bitmap1.Size(); i = bitmap1.NextNonZeroBit(i)) {
- bitmap2.Set(i);
- }
- CHECK_BITMAP(bitmap2, initBits2);
- UNIT_ASSERT_EQUAL(bitmap1, bitmap2);
-
- UNIT_ASSERT_EQUAL(bitmap1.NextNonZeroBit(63), bitmap1.Size());
- UNIT_ASSERT_EQUAL(bitmap1.NextNonZeroBit(64), bitmap1.Size());
- UNIT_ASSERT_EQUAL(bitmap1.NextNonZeroBit(65), bitmap1.Size());
- UNIT_ASSERT_EQUAL(bitmap1.NextNonZeroBit(127), bitmap1.Size());
- UNIT_ASSERT_EQUAL(bitmap1.NextNonZeroBit(533), bitmap1.Size());
-
- TBitMap<128, ui8> bitmap3;
- bitmap1.Clear();
- INIT_BITMAP(bitmap1, initBits1);
- for (size_t i = bitmap1.FirstNonZeroBit(); i != bitmap1.Size(); i = bitmap1.NextNonZeroBit(i)) {
- bitmap3.Set(i);
- }
- CHECK_BITMAP(bitmap3, initBits1);
- UNIT_ASSERT_EQUAL(bitmap3, bitmap1);
-
- TBitMap<18> bitmap4;
- bitmap4.Set(15);
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(0), 15);
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(15), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(63), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(64), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(65), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(127), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(533), bitmap4.Size());
-
- bitmap4.Clear().Flip();
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(0), 1);
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(15), 16);
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(17), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(18), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(63), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(64), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(65), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(127), bitmap4.Size());
- UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(533), bitmap4.Size());
- }
+ TDynBitMap bitmap1;
+ TDynBitMap bitmap2;
+
+ size_t initBits1[] = {0, 3, 7, 8, 11, 33, 34, 35, 36, 62, 63, 100, 127};
+ INIT_BITMAP(bitmap1, initBits1);
+ for (size_t i = bitmap1.FirstNonZeroBit(); i != bitmap1.Size(); i = bitmap1.NextNonZeroBit(i)) {
+ bitmap2.Set(i);
+ }
+ CHECK_BITMAP(bitmap2, initBits1);
+ UNIT_ASSERT_EQUAL(bitmap1, bitmap2);
+
+ size_t initBits2[] = {1, 2, 3, 4, 5, 6, 7, 8, 9, 10, 11, 33, 34, 35, 36, 62};
+ bitmap1.Clear();
+ bitmap2.Clear();
+ INIT_BITMAP(bitmap1, initBits2);
+ for (size_t i = bitmap1.FirstNonZeroBit(); i != bitmap1.Size(); i = bitmap1.NextNonZeroBit(i)) {
+ bitmap2.Set(i);
+ }
+ CHECK_BITMAP(bitmap2, initBits2);
+ UNIT_ASSERT_EQUAL(bitmap1, bitmap2);
+
+ UNIT_ASSERT_EQUAL(bitmap1.NextNonZeroBit(63), bitmap1.Size());
+ UNIT_ASSERT_EQUAL(bitmap1.NextNonZeroBit(64), bitmap1.Size());
+ UNIT_ASSERT_EQUAL(bitmap1.NextNonZeroBit(65), bitmap1.Size());
+ UNIT_ASSERT_EQUAL(bitmap1.NextNonZeroBit(127), bitmap1.Size());
+ UNIT_ASSERT_EQUAL(bitmap1.NextNonZeroBit(533), bitmap1.Size());
+
+ TBitMap<128, ui8> bitmap3;
+ bitmap1.Clear();
+ INIT_BITMAP(bitmap1, initBits1);
+ for (size_t i = bitmap1.FirstNonZeroBit(); i != bitmap1.Size(); i = bitmap1.NextNonZeroBit(i)) {
+ bitmap3.Set(i);
+ }
+ CHECK_BITMAP(bitmap3, initBits1);
+ UNIT_ASSERT_EQUAL(bitmap3, bitmap1);
+
+ TBitMap<18> bitmap4;
+ bitmap4.Set(15);
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(0), 15);
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(15), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(63), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(64), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(65), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(127), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(533), bitmap4.Size());
+
+ bitmap4.Clear().Flip();
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(0), 1);
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(15), 16);
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(17), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(18), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(63), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(64), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(65), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(127), bitmap4.Size());
+ UNIT_ASSERT_EQUAL(bitmap4.NextNonZeroBit(533), bitmap4.Size());
+ }
Y_UNIT_TEST(TestHashFixed) {
TBitMap<32, ui8> bitmap32;
@@ -550,27 +550,27 @@ Y_UNIT_TEST_SUITE(TBitMapTest) {
}
Y_UNIT_TEST(TestSetResetRange) {
- // Single chunk
+ // Single chunk
using TBitMap1Chunk = TBitMap<64>;
- UNIT_ASSERT_EQUAL(TBitMap1Chunk().Flip().Reset(10, 50), TBitMap1Chunk().Set(0, 10).Set(50, 64));
- UNIT_ASSERT_EQUAL(TBitMap1Chunk().Flip().Reset(0, 10), TBitMap1Chunk().Set(10, 64));
- UNIT_ASSERT_EQUAL(TBitMap1Chunk().Flip().Reset(50, 64), TBitMap1Chunk().Set(0, 50));
- UNIT_ASSERT_EQUAL(TBitMap1Chunk().Flip().Reset(0, 10).Reset(50, 64), TBitMap1Chunk().Set(10, 50));
-
- // Two chunks
+ UNIT_ASSERT_EQUAL(TBitMap1Chunk().Flip().Reset(10, 50), TBitMap1Chunk().Set(0, 10).Set(50, 64));
+ UNIT_ASSERT_EQUAL(TBitMap1Chunk().Flip().Reset(0, 10), TBitMap1Chunk().Set(10, 64));
+ UNIT_ASSERT_EQUAL(TBitMap1Chunk().Flip().Reset(50, 64), TBitMap1Chunk().Set(0, 50));
+ UNIT_ASSERT_EQUAL(TBitMap1Chunk().Flip().Reset(0, 10).Reset(50, 64), TBitMap1Chunk().Set(10, 50));
+
+ // Two chunks
using TBitMap2Chunks = TBitMap<64, ui32>;
- UNIT_ASSERT_EQUAL(TBitMap2Chunks().Flip().Reset(10, 50), TBitMap2Chunks().Set(0, 10).Set(50, 64));
- UNIT_ASSERT_EQUAL(TBitMap2Chunks().Flip().Reset(0, 10), TBitMap2Chunks().Set(10, 64));
- UNIT_ASSERT_EQUAL(TBitMap2Chunks().Flip().Reset(50, 64), TBitMap2Chunks().Set(0, 50));
- UNIT_ASSERT_EQUAL(TBitMap2Chunks().Flip().Reset(0, 10).Reset(50, 64), TBitMap2Chunks().Set(10, 50));
-
- // Many chunks
+ UNIT_ASSERT_EQUAL(TBitMap2Chunks().Flip().Reset(10, 50), TBitMap2Chunks().Set(0, 10).Set(50, 64));
+ UNIT_ASSERT_EQUAL(TBitMap2Chunks().Flip().Reset(0, 10), TBitMap2Chunks().Set(10, 64));
+ UNIT_ASSERT_EQUAL(TBitMap2Chunks().Flip().Reset(50, 64), TBitMap2Chunks().Set(0, 50));
+ UNIT_ASSERT_EQUAL(TBitMap2Chunks().Flip().Reset(0, 10).Reset(50, 64), TBitMap2Chunks().Set(10, 50));
+
+ // Many chunks
using TBitMap4Chunks = TBitMap<64, ui16>;
- UNIT_ASSERT_EQUAL(TBitMap4Chunks().Flip().Reset(10, 50), TBitMap4Chunks().Set(0, 10).Set(50, 64));
- UNIT_ASSERT_EQUAL(TBitMap4Chunks().Flip().Reset(0, 10), TBitMap4Chunks().Set(10, 64));
- UNIT_ASSERT_EQUAL(TBitMap4Chunks().Flip().Reset(50, 64), TBitMap4Chunks().Set(0, 50));
- UNIT_ASSERT_EQUAL(TBitMap4Chunks().Flip().Reset(0, 10).Reset(50, 64), TBitMap4Chunks().Set(10, 50));
- }
+ UNIT_ASSERT_EQUAL(TBitMap4Chunks().Flip().Reset(10, 50), TBitMap4Chunks().Set(0, 10).Set(50, 64));
+ UNIT_ASSERT_EQUAL(TBitMap4Chunks().Flip().Reset(0, 10), TBitMap4Chunks().Set(10, 64));
+ UNIT_ASSERT_EQUAL(TBitMap4Chunks().Flip().Reset(50, 64), TBitMap4Chunks().Set(0, 50));
+ UNIT_ASSERT_EQUAL(TBitMap4Chunks().Flip().Reset(0, 10).Reset(50, 64), TBitMap4Chunks().Set(10, 50));
+ }
Y_UNIT_TEST(TestSetRangeDyn) {
for (size_t start = 0; start < 192; ++start) {
@@ -594,4 +594,4 @@ Y_UNIT_TEST_SUITE(TBitMapTest) {
UNIT_ASSERT_VALUES_EQUAL(bm.Get(k), k >= 1 && k < 2048 ? 0 : 1);
}
}
-}
+}
diff --git a/util/generic/ptr.h b/util/generic/ptr.h
index 19db0e3ec5..31282045ff 100644
--- a/util/generic/ptr.h
+++ b/util/generic/ptr.h
@@ -102,7 +102,7 @@ template <class Base, class T>
class TPointerCommon {
public:
using TValueType = T;
-
+
inline T* operator->() const noexcept {
T* ptr = AsT();
Y_ASSERT(ptr);
diff --git a/util/stream/debug.cpp b/util/stream/debug.cpp
index afd5b3e1c7..6a43700e59 100644
--- a/util/stream/debug.cpp
+++ b/util/stream/debug.cpp
@@ -27,9 +27,9 @@ namespace {
}
} else {
Out = &Cnull;
- Level = 0;
- }
- }
+ Level = 0;
+ }
+ }
IOutputStream* Out;
int Level;
@@ -44,7 +44,7 @@ struct TSingletonTraits<TDbgSelector> {
IOutputStream& StdDbgStream() noexcept {
return *(Singleton<TDbgSelector>()->Out);
}
-
+
int StdDbgLevel() noexcept {
- return Singleton<TDbgSelector>()->Level;
-}
+ return Singleton<TDbgSelector>()->Level;
+}
diff --git a/util/stream/debug.h b/util/stream/debug.h
index 92d6d4b42d..123fdaefda 100644
--- a/util/stream/debug.h
+++ b/util/stream/debug.h
@@ -40,7 +40,7 @@ IOutputStream& StdDbgStream() noexcept;
* @see DBGTRACE
*/
int StdDbgLevel() noexcept;
-
+
/**
* Standard debug stream.
*
diff --git a/util/stream/trace.h b/util/stream/trace.h
index e74b6ecf3e..15720416da 100644
--- a/util/stream/trace.h
+++ b/util/stream/trace.h
@@ -1,7 +1,7 @@
-#pragma once
-
-#include "debug.h"
-
+#pragma once
+
+#include "debug.h"
+
/**
* Debug level, as set via `DBGOUT` environment variable.
*/
@@ -13,14 +13,14 @@ enum ETraceLevel: ui8 {
TRACE_DEBUG = 5,
TRACE_DETAIL = 6,
TRACE_VERBOSE = 7
-};
-
+};
+
#if !defined(NDEBUG) && !defined(Y_ENABLE_TRACE)
#define Y_ENABLE_TRACE
-#endif
-
+#endif
+
#ifdef Y_ENABLE_TRACE
-
+
/**
* Writes the given data into standard debug stream if current debug level set
* via `DBGOUT` environment variable permits it.
@@ -47,14 +47,14 @@ enum ETraceLevel: ui8 {
StdDbgStream() << args << Endl; \
} \
while (false)
-
-#else
-
+
+#else
+
#define Y_DBGTRACE(elevel, args) \
do { \
} while (false)
#define Y_DBGTRACE0(level, args) \
do { \
} while (false)
-
-#endif
+
+#endif
diff --git a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp
index 76f38bf35c..db5040406a 100644
--- a/ydb/core/client/minikql_compile/yql_expr_minikql.cpp
+++ b/ydb/core/client/minikql_compile/yql_expr_minikql.cpp
@@ -424,7 +424,7 @@ private:
auto selectType = GetSelectType(lookup, *selectTuple, ctx);
auto optSelectType = ctx.MakeType<TOptionalExprType>(selectType);
- node.SetTypeAnn(optSelectType);
+ node.SetTypeAnn(optSelectType);
return TStatus::Ok;
}
@@ -542,7 +542,7 @@ private:
resultItems.push_back(ctx.MakeType<TItemExprType>("Truncated", boolType));
auto resultType = ctx.MakeType<TStructExprType>(resultItems);
- node.SetTypeAnn(resultType);
+ node.SetTypeAnn(resultType);
return TStatus::Ok;
}
@@ -579,7 +579,7 @@ private:
}
}
- node.SetTypeAnn(ctx.MakeType<TVoidExprType>());
+ node.SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
@@ -591,7 +591,7 @@ private:
return TStatus::Error;
}
- node.SetTypeAnn(ctx.MakeType<TVoidExprType>());
+ node.SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
@@ -599,7 +599,7 @@ private:
Y_ENSURE_EX(node.ChildrenSize() == 2, TNodeException(node) << "SetResult expects 2 args.");
Y_ENSURE_EX(node.Child(0)->Type() == TExprNode::Atom, TNodeException(node) << "First SetResult argument should be Atom.");
- node.SetTypeAnn(ctx.MakeType<TVoidExprType>());
+ node.SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
@@ -608,7 +608,7 @@ private:
return IGraphTransformer::TStatus::Error;
}
- node.SetTypeAnn(node.Child(0)->GetTypeAnn());
+ node.SetTypeAnn(node.Child(0)->GetTypeAnn());
return TStatus::Ok;
}
@@ -625,7 +625,7 @@ private:
auto tupleType = ctx.MakeType<TTupleExprType>(items);
- node.SetTypeAnn(tupleType);
+ node.SetTypeAnn(tupleType);
return TStatus::Ok;
}
@@ -638,9 +638,9 @@ private:
return IGraphTransformer::TStatus::Error;
}
- auto parameterType = node.Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto parameterType = node.Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
- node.SetTypeAnn(parameterType);
+ node.SetTypeAnn(parameterType);
return TStatus::Ok;
}
@@ -649,7 +649,7 @@ private:
auto structType = ctx.MakeType<TStructExprType>(TVector<const TItemExprType*>());
- node.SetTypeAnn(structType);
+ node.SetTypeAnn(structType);
return TStatus::Ok;
}
@@ -661,7 +661,7 @@ private:
auto output = tmpNode;
YQL_ENSURE(CallableTransformer->Transform(tmpNode, output, ctx) == TStatus::Ok);
- node.SetTypeAnn(tmpNode->GetTypeAnn());
+ node.SetTypeAnn(tmpNode->GetTypeAnn());
return TStatus::Ok;
}
@@ -673,7 +673,7 @@ private:
auto output = tmpNode;
YQL_ENSURE(CallableTransformer->Transform(tmpNode, output, ctx) == TStatus::Ok);
- node.SetTypeAnn(tmpNode->GetTypeAnn());
+ node.SetTypeAnn(tmpNode->GetTypeAnn());
return TStatus::Ok;
}
@@ -691,20 +691,20 @@ private:
}
auto& lambda = node.ChildRef(1);
- const TTypeAnnotationNode* itemType = node.Child(0)->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ const TTypeAnnotationNode* itemType = node.Child(0)->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
if (!UpdateLambdaAllArgumentsTypes(lambda, {itemType}, ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!EnsureComputableType(lambda->Pos(), *lambda->GetTypeAnn(), ctx)) {
+ if (!EnsureComputableType(lambda->Pos(), *lambda->GetTypeAnn(), ctx)) {
return IGraphTransformer::TStatus::Error;
}
- node.SetTypeAnn(ctx.MakeType<TListExprType>(lambda->GetTypeAnn()));
+ node.SetTypeAnn(ctx.MakeType<TListExprType>(lambda->GetTypeAnn()));
return TStatus::Ok;
}
@@ -722,20 +722,20 @@ private:
}
auto& lambda = node.ChildRef(1);
- const TTypeAnnotationNode* itemType = node.Child(0)->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ const TTypeAnnotationNode* itemType = node.Child(0)->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
if (!UpdateLambdaAllArgumentsTypes(lambda, {itemType}, ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!EnsureListType(lambda->Pos(), *lambda->GetTypeAnn(), ctx)) {
+ if (!EnsureListType(lambda->Pos(), *lambda->GetTypeAnn(), ctx)) {
return IGraphTransformer::TStatus::Error;
}
- node.SetTypeAnn(lambda->GetTypeAnn());
+ node.SetTypeAnn(lambda->GetTypeAnn());
return TStatus::Ok;
}
@@ -747,7 +747,7 @@ private:
return IGraphTransformer::TStatus::Error;
}
- node.SetTypeAnn(ctx.MakeType<TVoidExprType>());
+ node.SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
@@ -756,7 +756,7 @@ private:
return IGraphTransformer::TStatus::Error;
}
- node.SetTypeAnn(ctx.MakeType<TVoidExprType>());
+ node.SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
@@ -764,11 +764,11 @@ private:
Y_UNUSED(ctx);
if (node.Content() == "ByteString") {
- node.SetTypeAnn(ctx.MakeType<TDataExprType>(EDataSlot::String));
+ node.SetTypeAnn(ctx.MakeType<TDataExprType>(EDataSlot::String));
} else if (node.Content() == "Utf8String") {
- node.SetTypeAnn(ctx.MakeType<TDataExprType>(EDataSlot::Utf8));
+ node.SetTypeAnn(ctx.MakeType<TDataExprType>(EDataSlot::Utf8));
} else {
- node.SetTypeAnn(ctx.MakeType<TDataExprType>(NKikimr::NUdf::GetDataSlot(node.Content())));
+ node.SetTypeAnn(ctx.MakeType<TDataExprType>(NKikimr::NUdf::GetDataSlot(node.Content())));
}
return TStatus::Ok;
@@ -778,12 +778,12 @@ private:
{
switch (node.ChildrenSize()) {
case 1:
- node.SetTypeAnn(ctx.MakeType<TTypeExprType>(ctx.MakeType<TDataExprType>(NKikimr::NUdf::GetDataSlot(
- AdaptLegacyYqlType(node.Child(0)->Content())))));
+ node.SetTypeAnn(ctx.MakeType<TTypeExprType>(ctx.MakeType<TDataExprType>(NKikimr::NUdf::GetDataSlot(
+ AdaptLegacyYqlType(node.Child(0)->Content())))));
break;
case 3:
- node.SetTypeAnn(ctx.MakeType<TTypeExprType>(ctx.MakeType<TDataExprParamsType>(
- NKikimr::NUdf::GetDataSlot(AdaptLegacyYqlType(node.Child(0)->Content())), node.Child(1)->Content(), node.Child(2)->Content())));
+ node.SetTypeAnn(ctx.MakeType<TTypeExprType>(ctx.MakeType<TDataExprParamsType>(
+ NKikimr::NUdf::GetDataSlot(AdaptLegacyYqlType(node.Child(0)->Content())), node.Child(1)->Content(), node.Child(2)->Content())));
break;
default:
return TStatus::Error;
@@ -1200,7 +1200,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr
return mkqlContext->PgmBuilder->MapParameter(list, [&](TRuntimeNode item) {
TMkqlBuildContext::TArgumentsMap innerArguments;
auto arg = node.Child(1)->Child(0)->Child(0);
- innerArguments[arg] = item;
+ innerArguments[arg] = item;
TMkqlBuildContext innerCtx(ctx, std::move(innerArguments), node.Child(1)->UniqueId());
auto res = MkqlBuildExpr(*node.Child(1)->Child(1), innerCtx);
return res;
@@ -1213,7 +1213,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr
return mkqlContext->PgmBuilder->FlatMapParameter(list, [&](TRuntimeNode item) {
TMkqlBuildContext::TArgumentsMap innerArguments;
auto arg = node.Child(1)->Child(0)->Child(0);
- innerArguments[arg] = item;
+ innerArguments[arg] = item;
TMkqlBuildContext innerCtx(ctx, std::move(innerArguments), node.Child(1)->UniqueId());
auto res = MkqlBuildExpr(*node.Child(1)->Child(1), innerCtx);
return res;
@@ -1238,7 +1238,7 @@ TIntrusivePtr<NCommon::IMkqlCallableCompiler> CreateMkqlCompiler(TContext::TPtr
return mkqlContext->PgmBuilder->PartialSort(list, ascending, [&](TRuntimeNode item) {
TMkqlBuildContext::TArgumentsMap innerArguments;
auto argItem = node.Child(2)->Child(0)->Child(0);
- innerArguments[argItem] = item;
+ innerArguments[argItem] = item;
TMkqlBuildContext innerCtx(ctx, std::move(innerArguments), node.Child(2)->UniqueId());
auto res = MkqlBuildExpr(*node.Child(2)->Child(1), innerCtx);
return res;
@@ -1569,7 +1569,7 @@ private:
return false;
}
Expr->Context.IssueManager.AddIssues(std::move(astRes.Issues));
- if (!CompileExpr(*root, Expr->Root, Expr->Context, nullptr)) {
+ if (!CompileExpr(*root, Expr->Root, Expr->Context, nullptr)) {
errors = Expr->Context.IssueManager.GetIssues();
return false;
}
diff --git a/ydb/core/engine/minikql/flat_local_tx_minikql.h b/ydb/core/engine/minikql/flat_local_tx_minikql.h
index 0feabf20e6..59c5f3fef7 100644
--- a/ydb/core/engine/minikql/flat_local_tx_minikql.h
+++ b/ydb/core/engine/minikql/flat_local_tx_minikql.h
@@ -94,7 +94,7 @@ class TFlatLocalMiniKQL : public NTabletFlatExecutor::ITransaction {
}
expr.Context.IssueManager.AddIssues(astResult.Issues);
- if (!NYql::CompileExpr(*astResult.Root, expr.Root, expr.Context, nullptr)) {
+ if (!NYql::CompileExpr(*astResult.Root, expr.Root, expr.Context, nullptr)) {
errors = expr.Context.IssueManager.GetIssues();
return false;
}
diff --git a/ydb/core/engine/mkql_engine_flat.cpp b/ydb/core/engine/mkql_engine_flat.cpp
index c7fe388eef..6b8291358a 100644
--- a/ydb/core/engine/mkql_engine_flat.cpp
+++ b/ydb/core/engine/mkql_engine_flat.cpp
@@ -241,7 +241,7 @@ public:
TEngineFlat(const TEngineFlatSettings& settings)
: Settings(settings)
, Alloc(Settings.AllocCounters, settings.FunctionRegistry->SupportsSizedAllocators())
- , Env(Alloc)
+ , Env(Alloc)
, Strings(Env)
, NeedDiagnostics(false)
, Status(EStatus::Unknown)
@@ -1022,7 +1022,7 @@ public:
TComputationPatternOpts opts(Alloc.Ref(), Env,
GetFlatShardExecutionFactory(execData, true),
Settings.FunctionRegistry,
- NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception,
+ NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception,
Settings.LlvmRuntime ? "" : "OFF", EGraphPerProcess::Multi);
auto pattern = MakeComputationPattern(explorer, runPgm, {}, opts);
auto graph = pattern->Clone(opts.ToComputationOptions(Settings.RandomProvider, Settings.TimeProvider));
diff --git a/ydb/core/kqp/host/kqp_host.cpp b/ydb/core/kqp/host/kqp_host.cpp
index 44af6d1168..1df3962b2e 100644
--- a/ydb/core/kqp/host/kqp_host.cpp
+++ b/ydb/core/kqp/host/kqp_host.cpp
@@ -540,10 +540,10 @@ public:
auto ret = input;
TExprBase node(input);
- if (auto maybeParameter = node.Maybe<TCoParameter>()) {
+ if (auto maybeParameter = node.Maybe<TCoParameter>()) {
auto parameter = maybeParameter.Cast();
auto name = parameter.Name().Value();
- auto expectedType = parameter.Ref().GetTypeAnn();
+ auto expectedType = parameter.Ref().GetTypeAnn();
if (queryCtx->PrepareOnly) {
auto& paramDesc = *queryCtx->PreparingQuery->AddParameters();
@@ -1045,8 +1045,8 @@ public:
FillSettings.Format = IDataProvider::EResultFormat::Custom;
FillSettings.FormatDetails = TString(KikimrMkqlProtoFormat);
- TypesCtx->AddDataSource(KikimrProviderName, kikimrDataSource);
- TypesCtx->AddDataSink(KikimrProviderName, kikimrDataSink);
+ TypesCtx->AddDataSource(KikimrProviderName, kikimrDataSource);
+ TypesCtx->AddDataSink(KikimrProviderName, kikimrDataSink);
TypesCtx->UdfResolver = CreateSimpleUdfResolver(FuncRegistry);
TypesCtx->TimeProvider = TAppData::TimeProvider;
TypesCtx->RandomProvider = TAppData::RandomProvider;
@@ -1059,7 +1059,7 @@ public:
ResultProviderConfig = MakeIntrusive<TResultProviderConfig>(*TypesCtx, *FuncRegistry, FillSettings.Format,
FillSettings.FormatDetails, writerFactory);
auto resultProvider = CreateResultProvider(ResultProviderConfig);
- TypesCtx->AddDataSink(ResultProviderName, resultProvider);
+ TypesCtx->AddDataSink(ResultProviderName, resultProvider);
TypesCtx->AvailablePureResultDataSources = TVector<TString>(1, TString(KikimrProviderName));
// Config provider
@@ -1076,16 +1076,16 @@ public:
.AddServiceTransformers()
.Add(TLogExprTransformer::Sync("YqlTransformer", NYql::NLog::EComponent::ProviderKqp,
NYql::NLog::ELevel::TRACE), "LogYqlTransform")
- .AddPreTypeAnnotation()
+ .AddPreTypeAnnotation()
// TODO: .AddExpressionEvaluation(*FuncRegistry)
.Add(new TFailExpressionEvaluation(), "FailExpressionEvaluation")
- .AddIOAnnotation()
- .AddTypeAnnotation()
+ .AddIOAnnotation()
+ .AddTypeAnnotation()
.Add(TCollectParametersTransformer::Sync(SessionCtx->QueryPtr()), "CollectParameters")
- .AddPostTypeAnnotation()
+ .AddPostTypeAnnotation()
.AddOptimization(true, false)
.Add(TLogExprTransformer::Sync("Optimized expr"), "LogExpr")
- .AddRun(&NullProgressWriter)
+ .AddRun(&NullProgressWriter)
.Build();
DataQueryAstTransformer = TTransformationPipeline(TypesCtx)
@@ -1456,7 +1456,7 @@ private:
}
TExprNode::TPtr result;
- if (!CompileExpr(*astRes.Root, result, ctx, ModuleResolver.get())) {
+ if (!CompileExpr(*astRes.Root, result, ctx, ModuleResolver.get())) {
return nullptr;
}
@@ -1477,11 +1477,11 @@ private:
return queryExpr;
}
- if (TMaybeNode<TCoCommit>(queryExpr) && TCoCommit(queryExpr).DataSink().Maybe<TKiDataSink>()) {
+ if (TMaybeNode<TCoCommit>(queryExpr) && TCoCommit(queryExpr).DataSink().Maybe<TKiDataSink>()) {
return queryExpr;
}
- return Build<TCoCommit>(ctx, queryExpr->Pos())
+ return Build<TCoCommit>(ctx, queryExpr->Pos())
.World(queryExpr)
.DataSink<TKiDataSink>()
.Category().Build(KikimrProviderName)
@@ -1490,7 +1490,7 @@ private:
.Settings()
.Add()
.Name().Build("mode")
- .Value<TCoAtom>().Build(KikimrCommitModeFlush())
+ .Value<TCoAtom>().Build(KikimrCommitModeFlush())
.Build()
.Build()
.Done()
diff --git a/ydb/core/kqp/host/kqp_run_prepared.cpp b/ydb/core/kqp/host/kqp_run_prepared.cpp
index e71a3bacee..dc56d47feb 100644
--- a/ydb/core/kqp/host/kqp_run_prepared.cpp
+++ b/ydb/core/kqp/host/kqp_run_prepared.cpp
@@ -153,7 +153,7 @@ private:
return false;
}
- return CompileExpr(*astRes.Root, expr, ctx, nullptr);
+ return CompileExpr(*astRes.Root, expr, ctx, nullptr);
}
bool ShouldAcquireLocks(const NKikimrKqp::TPreparedKql& kql) {
diff --git a/ydb/core/kqp/host/kqp_runner.cpp b/ydb/core/kqp/host/kqp_runner.cpp
index 42b4eca7c6..5a444df313 100644
--- a/ydb/core/kqp/host/kqp_runner.cpp
+++ b/ydb/core/kqp/host/kqp_runner.cpp
@@ -163,7 +163,7 @@ public:
.AddServiceTransformers()
.Add(TLogExprTransformer::Sync("KqlOptimizeTransformer", logComp, logLevel), "LogKqlOptimize")
.AddTypeAnnotationTransformer()
- .AddPostTypeAnnotation()
+ .AddPostTypeAnnotation()
.AddOptimization(false)
.Build(false);
@@ -198,7 +198,7 @@ public:
.AddTypeAnnotationTransformer(CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(),
*typesCtx, Config))
.Add(CreateKqpCheckQueryTransformer(), "CheckKqlQuery")
- .AddPostTypeAnnotation()
+ .AddPostTypeAnnotation()
.AddCommonOptimization()
.Add(CreateKqpLogOptTransformer(OptimizeCtx, *typesCtx, Config), "LogicalOptimize")
.Add(CreateKqpPhyOptTransformer(OptimizeCtx, *typesCtx), "PhysicalOptimize")
@@ -212,7 +212,7 @@ public:
.AddServiceTransformers()
.Add(TLogExprTransformer::Sync("PhysicalBuildQueryTransformer", logComp, logLevel), "LogPhysicalBuildQuery")
.AddTypeAnnotationTransformer(CreateKqpTypeAnnotationTransformer(Cluster, sessionCtx->TablesPtr(), *typesCtx, Config))
- .AddPostTypeAnnotation()
+ .AddPostTypeAnnotation()
.Add(
CreateKqpBuildTxsTransformer(
OptimizeCtx,
diff --git a/ydb/core/kqp/prepare/kqp_query_analyze.cpp b/ydb/core/kqp/prepare/kqp_query_analyze.cpp
index b37e4eb058..8c8f34ce79 100644
--- a/ydb/core/kqp/prepare/kqp_query_analyze.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_analyze.cpp
@@ -70,7 +70,7 @@ bool IsSafePayloadCallable(const TCallable& callable) {
if (auto maybeFilter = callable.Maybe<TCoFilterBase>()) {
auto filter = maybeFilter.Cast();
- if (filter.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
+ if (filter.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
return true;
}
@@ -84,7 +84,7 @@ bool IsSafePayloadCallable(const TCallable& callable) {
if (auto maybeMap = callable.Maybe<TCoMapBase>()) {
auto map = maybeMap.Cast();
- if (map.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
+ if (map.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
return true;
}
@@ -94,7 +94,7 @@ bool IsSafePayloadCallable(const TCallable& callable) {
} else {
auto body = map.Lambda().Body();
- return body.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional ||
+ return body.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional ||
body.Maybe<TCoToList>() || body.Maybe<TCoAsList>();
}
}
@@ -203,7 +203,7 @@ void RequireImmediate(TExprBase node, TAnalyzeTxContext& ctx) {
ctx.NodesInfo[node.Raw()].RequireImmediate = true;
}
-void RequireImmediateKey(TCoNameValueTupleList key, TAnalyzeTxContext& ctx) {
+void RequireImmediateKey(TCoNameValueTupleList key, TAnalyzeTxContext& ctx) {
for (auto tuple : key) {
YQL_ENSURE(tuple.Value().IsValid());
RequireImmediate(tuple.Value().Cast(), ctx);
@@ -313,8 +313,8 @@ void RequireEffectPayloadSafety(TCoNameValueTupleList key, TCoNameValueTupleList
}
void MarkImmediateNodes(TExprBase node, TAnalyzeTxContext& ctx) {
- if (node.Maybe<TCoDataType>() ||
- node.Maybe<TCoOptionalType>())
+ if (node.Maybe<TCoDataType>() ||
+ node.Maybe<TCoOptionalType>())
{
ctx.NodesInfo[node.Raw()].IsImmediate = true;
}
@@ -323,7 +323,7 @@ void MarkImmediateNodes(TExprBase node, TAnalyzeTxContext& ctx) {
ctx.NodesInfo[node.Raw()].IsImmediate = true;
}
- if (node.Maybe<TCoParameter>()) {
+ if (node.Maybe<TCoParameter>()) {
ctx.NodesInfo[node.Raw()].IsImmediate = true;
}
@@ -368,13 +368,13 @@ void MarkRequireImmediateNodes(TExprBase node, TAnalyzeTxContext& ctx) {
}
void PropagateImmediateNodes(TExprBase node, TAnalyzeTxContext& ctx) {
- if (auto just = node.Maybe<TCoJust>()) {
+ if (auto just = node.Maybe<TCoJust>()) {
if (ctx.NodesInfo[just.Cast().Input().Raw()].IsImmediate) {
ctx.NodesInfo[node.Raw()].IsImmediate = true;
}
}
- if (auto nth = node.Maybe<TCoNth>()) {
+ if (auto nth = node.Maybe<TCoNth>()) {
if (ctx.NodesInfo[nth.Cast().Tuple().Raw()].IsImmediate) {
ctx.NodesInfo[node.Raw()].IsImmediate = true;
}
@@ -440,7 +440,7 @@ void AnalyzeNode(TExprBase node, TAnalyzeTxContext& ctx) {
nodeInfo.IsExecutable = nodeInfo.IsExecutable && canExecute;
- if (!TMaybeNode<TCoLambda>(child)) {
+ if (!TMaybeNode<TCoLambda>(child)) {
nodeInfo.AreInputsExecutable = nodeInfo.AreInputsExecutable && canExecute;
}
diff --git a/ydb/core/kqp/prepare/kqp_query_exec.cpp b/ydb/core/kqp/prepare/kqp_query_exec.cpp
index bc2d4e94ce..eb26dabd21 100644
--- a/ydb/core/kqp/prepare/kqp_query_exec.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_exec.cpp
@@ -37,13 +37,13 @@ struct TParamBinding {
, ResultIndex(resultIndex) {}
};
-NKikimrKqp::TParameterBinding GetParameterBinding(TCoParameter param) {
+NKikimrKqp::TParameterBinding GetParameterBinding(TCoParameter param) {
auto name = TString(param.Name().Value());
NKikimrKqp::TParameterBinding binding;
binding.SetName(name);
if (param.Ref().HasResult()) {
- auto indexTuple = TCoAtomList(TExprNode::GetResult(param.Ptr()));
+ auto indexTuple = TCoAtomList(TExprNode::GetResult(param.Ptr()));
ui32 mkqlIndex = FromString<ui32>(indexTuple.Item(0).Value());
ui32 resultIndex = FromString<ui32>(indexTuple.Item(1).Value());
binding.SetMkqlIndex(mkqlIndex);
@@ -84,8 +84,8 @@ NKikimrMiniKQL::TParams BuildParamFromResult(const NKikimrKqp::TParameterBinding
bool GetPredicateValue(const TKiConditionalEffect& effect, const TKqlTransformContext& transformCtx,
const THashMap<TString, NKikimrKqp::TParameterBinding>& bindingsMap)
{
- YQL_ENSURE(effect.Predicate().Maybe<TCoParameter>());
- auto paramName = effect.Predicate().Cast<TCoParameter>().Name().Value();
+ YQL_ENSURE(effect.Predicate().Maybe<TCoParameter>());
+ auto paramName = effect.Predicate().Cast<TCoParameter>().Name().Value();
auto binding = bindingsMap.FindPtr(paramName);
YQL_ENSURE(binding);
@@ -808,7 +808,7 @@ void LogMkqlResult(const NKikimrMiniKQL::TResult& result, TExprContext& ctx) {
}
bool HasEffects(const TKiProgram& program) {
- return !program.Effects().Maybe<TCoList>();
+ return !program.Effects().Maybe<TCoList>();
}
bool HasResults(const TKiProgram& program) {
diff --git a/ydb/core/kqp/prepare/kqp_query_finalize.cpp b/ydb/core/kqp/prepare/kqp_query_finalize.cpp
index 5e442780f7..d35cee3a84 100644
--- a/ydb/core/kqp/prepare/kqp_query_finalize.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_finalize.cpp
@@ -36,7 +36,7 @@ TExprBase GetDeferredEffectsList(const TDeferredEffects& effects, TPositionHandl
effectNodes.push_back(effect.Node.Cast());
}
- return Build<TCoExtend>(ctx, pos)
+ return Build<TCoExtend>(ctx, pos)
.Add(effectNodes)
.Done();
}
@@ -56,14 +56,14 @@ TExprBase GetEraseLocksEffects(const TString& cluster, TPositionHandle pos, TCoP
.Key()
.Add()
.Name().Build("LockId")
- .Value<TCoMember>()
+ .Value<TCoMember>()
.Struct("lockItem")
.Name().Build("LockId")
.Build()
.Build()
.Add()
.Name().Build("DataShard")
- .Value<TCoMember>()
+ .Value<TCoMember>()
.Struct("lockItem")
.Name().Build("DataShard")
.Build()
@@ -418,7 +418,7 @@ private:
YQL_ENSURE(TxState->Tx().ParamsState->Values.emplace(std::make_pair(locksParamName,
GetLocksParamValue(TxState->Tx().Locks))).second);
- auto locksParamNode = Build<TCoParameter>(ctx, pos)
+ auto locksParamNode = Build<TCoParameter>(ctx, pos)
.Name().Build(locksParamName)
.Type(ExpandType(pos, *GetTxLockListType(ctx), ctx))
.Done();
@@ -441,7 +441,7 @@ private:
YQL_ENSURE(TxState->Tx().ParamsState->Values.emplace(std::make_pair(locksParamName,
GetLocksParamValue(TxState->Tx().Locks))).second);
- auto locksParamNode = Build<TCoParameter>(ctx, pos)
+ auto locksParamNode = Build<TCoParameter>(ctx, pos)
.Name().Build(locksParamName)
.Type(ExpandType(pos, *GetTxLockListType(ctx), ctx))
.Done();
@@ -449,7 +449,7 @@ private:
if (!hasDataEffects && TxState->Tx().GetSnapshot().IsValid())
return GetEraseLocksEffects(Cluster, pos, locksParamNode, ctx);
- auto lockArg = Build<TCoArgument>(ctx, pos)
+ auto lockArg = Build<TCoArgument>(ctx, pos)
.Name("lockArg")
.Done();
@@ -463,14 +463,14 @@ private:
.Key()
.Add()
.Name().Build("LockId")
- .Value<TCoMember>()
+ .Value<TCoMember>()
.Struct(lockArg)
.Name().Build("LockId")
.Build()
.Build()
.Add()
.Name().Build("DataShard")
- .Value<TCoMember>()
+ .Value<TCoMember>()
.Struct(lockArg)
.Name().Build("DataShard")
.Build()
@@ -518,12 +518,12 @@ private:
.Build()
.Done()
};
- auto lockPredicate = Build<TCoNot>(ctx, pos)
- .Value<TCoCoalesce>()
- .Predicate<TCoAnd>()
+ auto lockPredicate = Build<TCoNot>(ctx, pos)
+ .Value<TCoCoalesce>()
+ .Predicate<TCoAnd>()
.Add(args)
.Build()
- .Value<TCoBool>()
+ .Value<TCoBool>()
.Literal().Build("false")
.Build()
.Build()
@@ -544,10 +544,10 @@ private:
.List(locksInvalidatedList)
.Done();
- auto effects = Build<TCoExtend>(ctx, pos)
- .Add<TCoIf>()
+ auto effects = Build<TCoExtend>(ctx, pos)
+ .Add<TCoIf>()
.Predicate(locksInvalidatedPredicate)
- .ThenValue<TCoAsList>()
+ .ThenValue<TCoAsList>()
.Add<TKiSetResult>()
.Name().Build(LocksInvalidatedResultName)
.Data(locksInvalidatedPredicate)
@@ -604,9 +604,9 @@ private:
} // namespace
TCoList GetEmptyEffectsList(const TPositionHandle pos, TExprContext& ctx) {
- return Build<TCoList>(ctx, pos)
- .ListType<TCoListType>()
- .ItemType<TCoVoidType>()
+ return Build<TCoList>(ctx, pos)
+ .ListType<TCoListType>()
+ .ItemType<TCoVoidType>()
.Build()
.Build()
.Done();
diff --git a/ydb/core/kqp/prepare/kqp_query_rewrite.cpp b/ydb/core/kqp/prepare/kqp_query_rewrite.cpp
index cb24a1b8a1..f567b48252 100644
--- a/ydb/core/kqp/prepare/kqp_query_rewrite.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_rewrite.cpp
@@ -12,16 +12,16 @@ namespace {
template<typename TMapNode>
TExprBase RebuildMapToList(TMapNode map, TExprContext& ctx) {
- if (map.Lambda().Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List &&
- map.Input().Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List)
+ if (map.Lambda().Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List &&
+ map.Input().Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List)
{
return map;
}
- bool isOptional = map.Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional;
+ bool isOptional = map.Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional;
- if (map.Lambda().Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
- auto newBody = Build<TCoToList>(ctx, map.Pos())
+ if (map.Lambda().Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
+ auto newBody = Build<TCoToList>(ctx, map.Pos())
.Optional(map.Lambda().Body())
.Done();
@@ -37,9 +37,9 @@ TExprBase RebuildMapToList(TMapNode map, TExprContext& ctx) {
.Done();
}
- if (map.Input().Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
+ if (map.Input().Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
map = Build<TMapNode>(ctx, map.Pos())
- .template Input<TCoToList>()
+ .template Input<TCoToList>()
.Optional(map.Input())
.Build()
.Lambda(map.Lambda())
@@ -47,7 +47,7 @@ TExprBase RebuildMapToList(TMapNode map, TExprContext& ctx) {
}
if (isOptional) {
- return Build<TCoToOptional>(ctx, map.Pos())
+ return Build<TCoToOptional>(ctx, map.Pos())
.List(map)
.Done();
}
@@ -61,15 +61,15 @@ TExprNode::TPtr NormalizeCallables(TExprBase node, TExprContext& ctx, const TKqp
return node.Ptr();
}
- if (node.Maybe<TCoMap>() || node.Maybe<TCoFlatMap>()) {
- return node.Maybe<TCoMap>()
- ? RebuildMapToList<TCoMap>(node.Cast<TCoMap>(), ctx).Ptr()
- : RebuildMapToList<TCoFlatMap>(node.Cast<TCoFlatMap>(), ctx).Ptr();
+ if (node.Maybe<TCoMap>() || node.Maybe<TCoFlatMap>()) {
+ return node.Maybe<TCoMap>()
+ ? RebuildMapToList<TCoMap>(node.Cast<TCoMap>(), ctx).Ptr()
+ : RebuildMapToList<TCoFlatMap>(node.Cast<TCoFlatMap>(), ctx).Ptr();
}
- if (auto filter = node.Maybe<TCoFilter>()) {
+ if (auto filter = node.Maybe<TCoFilter>()) {
YQL_CLOG(INFO, ProviderKqp) << "NormalizeCallables: Filter";
- return Build<TCoFlatMap>(ctx, node.Pos())
+ return Build<TCoFlatMap>(ctx, node.Pos())
.Input(filter.Cast().Input())
.Lambda()
.Args({"item"})
@@ -100,8 +100,8 @@ TExprNode::TPtr NormalizeCallables(TExprBase node, TExprContext& ctx, const TKqp
TExprNode::TPtr ToListOverToOptional(TExprBase node, TExprContext& ctx) {
Y_UNUSED(ctx);
- if (auto toList = node.Maybe<TCoToList>()) {
- if (auto toOpt = toList.Cast().Optional().Maybe<TCoToOptional>()) {
+ if (auto toList = node.Maybe<TCoToList>()) {
+ if (auto toOpt = toList.Cast().Optional().Maybe<TCoToOptional>()) {
YQL_CLOG(INFO, ProviderKqp) << "ToListOverToOptional";
return toOpt.Cast().List().Ptr();
}
@@ -119,7 +119,7 @@ TExprNode::TPtr ToListOverToOptional(TExprBase node, TExprContext& ctx) {
}
template<typename TInner, typename TOuter>
-TExprBase SplitMap(TExprBase input, TCoLambda lambda, TExprContext& ctx, const TVector<TExprBase> execRoots,
+TExprBase SplitMap(TExprBase input, TCoLambda lambda, TExprContext& ctx, const TVector<TExprBase> execRoots,
const TKqpAnalyzeResults& analyzeResults)
{
auto exprRootsTuple = Build<TExprList>(ctx, lambda.Pos())
@@ -165,7 +165,7 @@ TExprBase SplitMap(TExprBase input, TCoLambda lambda, TExprContext& ctx, const T
auto* nodeInfo = analyzeResults.ExprToNodeInfoMap.FindPtr(node.Get());
YQL_ENSURE(nodeInfo);
- YQL_ENSURE(node->GetTypeAnn());
+ YQL_ENSURE(node->GetTypeAnn());
bool isComputable = node->IsComputable();
if (innerNodes.contains(node.Get())) {
@@ -186,8 +186,8 @@ TExprBase SplitMap(TExprBase input, TCoLambda lambda, TExprContext& ctx, const T
bool innerChild = innerNodes.contains(child.Get());
hasInnerChild = hasInnerChild | innerChild;
- YQL_ENSURE(child->GetTypeAnn());
- if (child->GetTypeAnn()->IsComputable() && !innerChild) {
+ YQL_ENSURE(child->GetTypeAnn());
+ if (child->GetTypeAnn()->IsComputable() && !innerChild) {
return true;
}
}
@@ -211,7 +211,7 @@ TExprBase SplitMap(TExprBase input, TCoLambda lambda, TExprContext& ctx, const T
.Add(jointNodes)
.Done();
- auto innerLambda = Build<TCoLambda>(ctx, lambda.Pos())
+ auto innerLambda = Build<TCoLambda>(ctx, lambda.Pos())
.Args({"item"})
.Body<TExprApplier>()
.Apply(innerLambdaBody)
@@ -219,13 +219,13 @@ TExprBase SplitMap(TExprBase input, TCoLambda lambda, TExprContext& ctx, const T
.Build()
.Done();
- TCoArgument outerLambdaArg = Build<TCoArgument>(ctx, lambda.Pos())
+ TCoArgument outerLambdaArg = Build<TCoArgument>(ctx, lambda.Pos())
.Name("item")
.Done();
TNodeOnNodeOwnedMap replaceMap;
for (size_t i = 0; i < jointNodes.size(); ++i) {
- auto node = Build<TCoNth>(ctx, lambda.Pos())
+ auto node = Build<TCoNth>(ctx, lambda.Pos())
.Tuple(outerLambdaArg)
.Index().Build(i)
.Done();
@@ -234,7 +234,7 @@ TExprBase SplitMap(TExprBase input, TCoLambda lambda, TExprContext& ctx, const T
}
auto outerLambdaBody = ctx.ReplaceNodes(lambda.Body().Ptr(), replaceMap);
- auto outerLambda = Build<TCoLambda>(ctx, lambda.Pos())
+ auto outerLambda = Build<TCoLambda>(ctx, lambda.Pos())
.Args({outerLambdaArg})
.Body(TExprBase(outerLambdaBody))
.Done();
@@ -251,18 +251,18 @@ TExprBase SplitMap(TExprBase input, TCoLambda lambda, TExprContext& ctx, const T
TExprNode::TPtr SplitMap(TExprBase mapNode, TExprContext& ctx, const TVector<TExprBase>& execRoots,
const TKqpAnalyzeResults& analyzeResults)
{
- YQL_ENSURE(mapNode.Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List);
+ YQL_ENSURE(mapNode.Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List);
- if (auto map = mapNode.Maybe<TCoMap>()) {
+ if (auto map = mapNode.Maybe<TCoMap>()) {
YQL_CLOG(INFO, ProviderKqp) << "SplitMap: Map, MapParameter";
- return SplitMap<TCoMap, TKiMapParameter>(map.Cast().Input(), map.Cast().Lambda(), ctx, execRoots,
+ return SplitMap<TCoMap, TKiMapParameter>(map.Cast().Input(), map.Cast().Lambda(), ctx, execRoots,
analyzeResults).Ptr();
}
- if (auto map = mapNode.Maybe<TCoFlatMap>()) {
+ if (auto map = mapNode.Maybe<TCoFlatMap>()) {
YQL_CLOG(INFO, ProviderKqp) << "SplitMap: Map, FlatMapParameter";
- YQL_ENSURE(map.Cast().Lambda().Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List);
- return SplitMap<TCoMap, TKiFlatMapParameter>(map.Cast().Input(), map.Cast().Lambda(), ctx, execRoots,
+ YQL_ENSURE(map.Cast().Lambda().Ptr()->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List);
+ return SplitMap<TCoMap, TKiFlatMapParameter>(map.Cast().Input(), map.Cast().Lambda(), ctx, execRoots,
analyzeResults).Ptr();
}
@@ -313,8 +313,8 @@ TExprNode::TPtr UnnestExecutionRoots(TExprBase node, TExprContext& ctx, const TK
}
}
- if (node.Maybe<TCoMap>() ||
- node.Maybe<TCoFlatMap>() ||
+ if (node.Maybe<TCoMap>() ||
+ node.Maybe<TCoFlatMap>() ||
node.Maybe<TKiMapParameter>() ||
node.Maybe<TKiFlatMapParameter>())
{
diff --git a/ydb/core/kqp/prepare/kqp_query_simplify.cpp b/ydb/core/kqp/prepare/kqp_query_simplify.cpp
index d8fefeb939..6fa885aff5 100644
--- a/ydb/core/kqp/prepare/kqp_query_simplify.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_simplify.cpp
@@ -13,7 +13,7 @@ using namespace NYql::NNodes;
namespace {
template<typename TMapNode>
-TExprBase BuildMap(TExprBase input, TExprBase body, TCoArgument arg, TExprContext& ctx) {
+TExprBase BuildMap(TExprBase input, TExprBase body, TCoArgument arg, TExprContext& ctx) {
return Build<TMapNode>(ctx, input.Pos())
.Input(input)
.Lambda()
@@ -27,18 +27,18 @@ TExprBase BuildMap(TExprBase input, TExprBase body, TCoArgument arg, TExprContex
}
TExprNode::TPtr ExtractFilter(TExprBase node, TExprContext& ctx) {
- if (!node.Maybe<TCoFlatMap>()) {
+ if (!node.Maybe<TCoFlatMap>()) {
return node.Ptr();
}
- auto flatmap = node.Cast<TCoFlatMap>();
+ auto flatmap = node.Cast<TCoFlatMap>();
auto flatmapArg = flatmap.Lambda().Args().Arg(0);
- if (flatmap.Input().Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::List) {
+ if (flatmap.Input().Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::List) {
return node.Ptr();
}
- if (auto maybeConditional = flatmap.Lambda().Body().Maybe<TCoConditionalValueBase>()) {
+ if (auto maybeConditional = flatmap.Lambda().Body().Maybe<TCoConditionalValueBase>()) {
auto conditional = maybeConditional.Cast();
auto blacklistedNode = FindNode(conditional.Predicate().Ptr(), [](const TExprNode::TPtr& exprNode) {
@@ -55,7 +55,7 @@ TExprNode::TPtr ExtractFilter(TExprBase node, TExprContext& ctx) {
return node.Ptr();
}
- auto filter = Build<TCoFilter>(ctx, node.Pos())
+ auto filter = Build<TCoFilter>(ctx, node.Pos())
.Input(flatmap.Input())
.Lambda()
.Args({"item"})
@@ -68,10 +68,10 @@ TExprNode::TPtr ExtractFilter(TExprBase node, TExprContext& ctx) {
auto value = conditional.Value();
- if (conditional.Maybe<TCoListIf>() || conditional.Maybe<TCoOptionalIf>()) {
- return BuildMap<TCoMap>(filter, value, flatmapArg, ctx).Ptr();
+ if (conditional.Maybe<TCoListIf>() || conditional.Maybe<TCoOptionalIf>()) {
+ return BuildMap<TCoMap>(filter, value, flatmapArg, ctx).Ptr();
} else {
- return BuildMap<TCoFlatMap>(filter, value, flatmapArg, ctx).Ptr();
+ return BuildMap<TCoFlatMap>(filter, value, flatmapArg, ctx).Ptr();
}
}
@@ -117,11 +117,11 @@ TExprNode::TPtr ExtractFilter(TExprBase node, TExprContext& ctx) {
}
TExprNode::TPtr ExtractCombineByKeyPreMap(TExprBase node, TExprContext& ctx) {
- if (auto maybeCombine = node.Maybe<TCoCombineByKey>()) {
+ if (auto maybeCombine = node.Maybe<TCoCombineByKey>()) {
auto combine = maybeCombine.Cast();
if (!IsKqlPureLambda(combine.PreMapLambda())) {
- return Build<TCoCombineByKey>(ctx, node.Pos())
- .Input<TCoMap>()
+ return Build<TCoCombineByKey>(ctx, node.Pos())
+ .Input<TCoMap>()
.Input(combine.Input())
.Lambda(combine.PreMapLambda())
.Build()
diff --git a/ydb/core/kqp/prepare/kqp_query_substitute.cpp b/ydb/core/kqp/prepare/kqp_query_substitute.cpp
index 6d289bf78b..dd43f7b0fc 100644
--- a/ydb/core/kqp/prepare/kqp_query_substitute.cpp
+++ b/ydb/core/kqp/prepare/kqp_query_substitute.cpp
@@ -32,16 +32,16 @@ public:
auto node = analyzeResults.ExecutionRoots[i].Node;
- YQL_ENSURE(node.Ref().GetTypeAnn());
- auto paramNode = Build<TCoParameter>(ctx, node.Pos())
+ YQL_ENSURE(node.Ref().GetTypeAnn());
+ auto paramNode = Build<TCoParameter>(ctx, node.Pos())
.Name().Build(newParamName)
- .Type(ExpandType(node.Pos(), *node.Ref().GetTypeAnn(), ctx))
+ .Type(ExpandType(node.Pos(), *node.Ref().GetTypeAnn(), ctx))
.Done();
YQL_ENSURE(!TransformCtx->MkqlResults.empty());
ui32 mkqlIndex = TransformCtx->MkqlResults.size() - 1;
ui32 resultIndex = i;
- auto indexTuple = Build<TCoAtomList>(ctx, paramNode.Pos())
+ auto indexTuple = Build<TCoAtomList>(ctx, paramNode.Pos())
.Add().Build(ToString(mkqlIndex))
.Add().Build(ToString(resultIndex))
.Done();
diff --git a/ydb/core/kqp/prepare/kqp_type_ann.cpp b/ydb/core/kqp/prepare/kqp_type_ann.cpp
index ee51c7349c..1cb4dc51d7 100644
--- a/ydb/core/kqp/prepare/kqp_type_ann.cpp
+++ b/ydb/core/kqp/prepare/kqp_type_ann.cpp
@@ -1155,7 +1155,7 @@ TAutoPtr<IGraphTransformer> CreateKqpCheckKiProgramTransformer() {
YQL_ENSURE(TMaybeNode<TKiProgram>(input));
auto program = TKiProgram(input);
- auto effectsType = program.Effects().Ptr()->GetTypeAnn();
+ auto effectsType = program.Effects().Ptr()->GetTypeAnn();
bool typeOk = EnsureListType(input->Pos(), *effectsType, ctx);
if (typeOk) {
auto listType = effectsType->Cast<TListExprType>();
diff --git a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
index 035d6b7701..1db0eceb6d 100644
--- a/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_datasink.cpp
@@ -2,7 +2,7 @@
#include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h>
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
-
+
#include <ydb/library/yql/core/yql_expr_optimize.h>
namespace NYql {
@@ -298,7 +298,7 @@ private:
TIntrusivePtr<TKikimrSessionContext> SessionCtx;
};
-class TKikimrDataSink : public TDataProviderBase
+class TKikimrDataSink : public TDataProviderBase
{
public:
TKikimrDataSink(
@@ -386,8 +386,8 @@ public:
return *PlanInfoTransformer;
}
- bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
- if (node.IsCallable(TCoDataSink::CallableName())) {
+ bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
+ if (node.IsCallable(TCoDataSink::CallableName())) {
if (node.Child(0)->Content() == KikimrProviderName) {
if (node.Child(1)->Content().empty()) {
ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), "Empty cluster name"));
@@ -404,7 +404,7 @@ public:
}
bool CanParse(const TExprNode& node) override {
- if (node.IsCallable(WriteName)) {
+ if (node.IsCallable(WriteName)) {
return node.Child(1)->Child(0)->Content() == KikimrProviderName;
}
@@ -754,7 +754,7 @@ IGraphTransformer::TStatus TKiSinkVisitorTransformer::DoTransform(TExprNode::TPt
return HandleWrite(TExprBase(input), ctx);
}
- if (auto node = callable.Maybe<TCoCommit>()) {
+ if (auto node = callable.Maybe<TCoCommit>()) {
return HandleCommit(node.Cast(), ctx);
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
index 65ef7eb932..b9f8a7510f 100644
--- a/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_datasource.cpp
@@ -2,7 +2,7 @@
#include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h>
#include <ydb/library/yql/providers/common/config/yql_configuration_transformer.h>
-
+
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
@@ -267,7 +267,7 @@ private:
TIntrusivePtr<TKikimrSessionContext> SessionCtx;
};
-class TKikimrDataSource : public TDataProviderBase {
+class TKikimrDataSource : public TDataProviderBase {
public:
TKikimrDataSource(
const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry,
@@ -361,8 +361,8 @@ public:
return *CallableExecutionTransformer;
}
- bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
- if (node.IsCallable(TCoDataSource::CallableName())) {
+ bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
+ if (node.IsCallable(TCoDataSource::CallableName())) {
if (node.Child(0)->Content() == KikimrProviderName) {
if (node.Child(1)->Content().empty()) {
ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), "Empty cluster name"));
@@ -379,7 +379,7 @@ public:
}
bool CanParse(const TExprNode& node) override {
- if (node.IsCallable(ReadName)) {
+ if (node.IsCallable(ReadName)) {
return node.Child(1)->Child(0)->Content() == KikimrProviderName;
}
@@ -392,22 +392,22 @@ public:
}
bool IsPersistent(const TExprNode& node) override {
- if (node.IsCallable(ReadName)) {
- return node.Child(1)->Child(0)->Content() == KikimrProviderName;
- }
-
- if (node.IsCallable(TKiReadTable::CallableName())) {
- return TKiDataSource(node.ChildPtr(1)).Category() == KikimrProviderName;
- }
-
- return false;
- }
-
+ if (node.IsCallable(ReadName)) {
+ return node.Child(1)->Child(0)->Content() == KikimrProviderName;
+ }
+
+ if (node.IsCallable(TKiReadTable::CallableName())) {
+ return TKiDataSource(node.ChildPtr(1)).Category() == KikimrProviderName;
+ }
+
+ return false;
+ }
+
bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) override {
Y_UNUSED(syncList);
canRef = false;
- if (node.IsCallable(TCoRight::CallableName())) {
+ if (node.IsCallable(TCoRight::CallableName())) {
const auto input = node.Child(0);
if (input->IsCallable(TKiReadTableList::CallableName())) {
return true;
@@ -432,7 +432,7 @@ public:
return true;
}
- if (auto configure = TMaybeNode<TCoConfigure>(&node)) {
+ if (auto configure = TMaybeNode<TCoConfigure>(&node)) {
if (configure.DataSource().Maybe<TKiDataSource>()) {
return true;
}
@@ -443,7 +443,7 @@ public:
TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override {
auto read = node->Child(0);
- if (!read->IsCallable(ReadName)) {
+ if (!read->IsCallable(ReadName)) {
ythrow yexception() << "Expected Read!";
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_exec.cpp b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
index ad360fcaeb..6355b950f6 100644
--- a/ydb/core/kqp/provider/yql_kikimr_exec.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_exec.cpp
@@ -190,7 +190,7 @@ public:
IDataProvider::TFillSettings fillSettings = NCommon::GetFillSettings(*input);
- if (auto maybeTableList = TMaybeNode<TCoRight>(pullInput).Input().Maybe<TKiReadTableList>()) {
+ if (auto maybeTableList = TMaybeNode<TCoRight>(pullInput).Input().Maybe<TKiReadTableList>()) {
if (!EnsureNotPrepare("tablelist", pullInput->Pos(), SessionCtx->Query(), ctx)) {
return SyncError();
}
@@ -210,7 +210,7 @@ public:
});
}
- if (auto maybeTableScheme = TMaybeNode<TCoRight>(pullInput).Input().Maybe<TKiReadTableScheme>()) {
+ if (auto maybeTableScheme = TMaybeNode<TCoRight>(pullInput).Input().Maybe<TKiReadTableScheme>()) {
if (!EnsureNotPrepare("tablescheme", pullInput->Pos(), SessionCtx->Query(), ctx)) {
return SyncError();
}
@@ -249,7 +249,7 @@ public:
auto exec = resultInput.Maybe<TCoNth>().Tuple().Maybe<TCoRight>().Input();
YQL_ENSURE(exec.Maybe<TKiExecDataQuery>());
- ui32 index = FromString<ui32>(resultInput.Cast<TCoNth>().Index().Value());
+ ui32 index = FromString<ui32>(resultInput.Cast<TCoNth>().Index().Value());
YQL_ENSURE(index == 0);
return RunResOrPullForExec(TResOrPullBase(input), exec.Cast(), ctx, 0);
@@ -261,7 +261,7 @@ public:
return SyncStatus(requireStatus);
}
- auto configure = TCoConfigure(input);
+ auto configure = TCoConfigure(input);
auto clusterName = TString(configure.DataSource().Arg(1).Cast<TCoAtom>().Value());
if (configure.Arg(2).Cast<TCoAtom>().Value() == TStringBuf("Attr")) {
auto name = TString(configure.Arg(3).Cast<TCoAtom>().Value());
@@ -383,7 +383,7 @@ private:
return SyncOk();
}
- auto executeRightType = exec.Ref().GetTypeAnn()->Cast<TTupleExprType>()->GetItems()[1];
+ auto executeRightType = exec.Ref().GetTypeAnn()->Cast<TTupleExprType>()->GetItems()[1];
auto resultType = executeRightType->Cast<TTupleExprType>()->GetItems()[resultIndex];
YQL_ENSURE(resultIndex < runResult->Results.size());
@@ -417,7 +417,7 @@ public:
YQL_ENSURE(input->Type() == TExprNode::Callable);
output = input;
- if (auto maybeCommit = TMaybeNode<TCoCommit>(input)) {
+ if (auto maybeCommit = TMaybeNode<TCoCommit>(input)) {
auto requireStatus = RequireChild(*input, 0);
if (requireStatus.Level != TStatus::Ok) {
return SyncStatus(requireStatus);
@@ -444,7 +444,7 @@ public:
return RunKiExecDataQuery(ctx, maybeExecQuery.Cast());
}
- if (TMaybeNode<TCoNth>(input)) {
+ if (TMaybeNode<TCoNth>(input)) {
input->SetState(TExprNode::EState::ExecutionComplete);
input->SetResult(ctx.NewWorld(input->Pos()));
return SyncOk();
diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.cpp b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.cpp
index cbe44c3a9d..e267ba7d6e 100644
--- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.cpp
@@ -16,11 +16,11 @@ TString TKiReadTable::GetTable(TExprContext& ctx) const {
TCoAtomList TKiReadTable::GetSelectColumns(TExprContext& ctx, const TKikimrTableDescription& tableData,
bool withSystemColumns) const
{
- if (Select().Maybe<TCoVoid>()) {
+ if (Select().Maybe<TCoVoid>()) {
return BuildColumnsList(tableData, Pos(), ctx, withSystemColumns);
}
- return Select().Cast<TCoAtomList>();
+ return Select().Cast<TCoAtomList>();
}
TCoAtomList TKiReadTable::GetSelectColumns(TExprContext& ctx, const TKikimrTablesData& tablesData,
diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h
index e91fd7635a..4f3713f12b 100644
--- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h
+++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.h
@@ -11,7 +11,7 @@ namespace NNodes {
#include <ydb/core/kqp/provider/yql_kikimr_expr_nodes.decl.inl.h>
-class TKiDataSource : public NGenerated::TKiDataSourceStub<TExprBase, TCallable, TCoAtom> {
+class TKiDataSource : public NGenerated::TKiDataSourceStub<TExprBase, TCallable, TCoAtom> {
public:
explicit TKiDataSource(const TExprNode* node)
: TKiDataSourceStub(node) {}
@@ -32,7 +32,7 @@ public:
}
};
-class TKiDataSink : public NGenerated::TKiDataSinkStub<TExprBase, TCallable, TCoAtom> {
+class TKiDataSink : public NGenerated::TKiDataSinkStub<TExprBase, TCallable, TCoAtom> {
public:
explicit TKiDataSink(const TExprNode* node)
: TKiDataSinkStub(node) {}
diff --git a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json
index c54d82a2ca..1bf5468d97 100644
--- a/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json
+++ b/ydb/core/kqp/provider/yql_kikimr_expr_nodes.json
@@ -11,8 +11,8 @@
"Definition": "Custom",
"Match": {"Type": "Callable", "Name": "DataSource"},
"Children": [
- {"Index": 0, "Name": "Category", "Type": "TCoAtom"},
- {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"}
+ {"Index": 0, "Name": "Category", "Type": "TCoAtom"},
+ {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"}
]
},
{
@@ -21,8 +21,8 @@
"Definition": "Custom",
"Match": {"Type": "Callable", "Name": "DataSink"},
"Children": [
- {"Index": 0, "Name": "Category", "Type": "TCoAtom"},
- {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"}
+ {"Index": 0, "Name": "Category", "Type": "TCoAtom"},
+ {"Index": 1, "Name": "Cluster", "Type": "TCoAtom"}
]
},
{
@@ -53,7 +53,7 @@
"Match": {"Type": "Callable", "Name": "KiReadTable!"},
"Children": [
{"Index": 3, "Name": "Select", "Type": "TExprBase"},
- {"Index": 4, "Name": "Settings", "Type": "TCoNameValueTupleList"}
+ {"Index": 4, "Name": "Settings", "Type": "TCoNameValueTupleList"}
]
},
{
@@ -73,10 +73,10 @@
"Children": [
{"Index": 0, "Name": "World", "Type": "TExprBase"},
{"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"},
- {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
{"Index": 3, "Name": "Input", "Type": "TExprBase"},
- {"Index": 4, "Name": "Mode", "Type": "TCoAtom"},
- {"Index": 5, "Name": "Settings", "Type": "TCoNameValueTupleList"}
+ {"Index": 4, "Name": "Mode", "Type": "TCoAtom"},
+ {"Index": 5, "Name": "Settings", "Type": "TCoNameValueTupleList"}
]
},
{
@@ -86,9 +86,9 @@
"Children": [
{"Index": 0, "Name": "World", "Type": "TExprBase"},
{"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"},
- {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
- {"Index": 3, "Name": "Filter", "Type": "TCoLambda"},
- {"Index": 4, "Name": "Update", "Type": "TCoLambda"}
+ {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
+ {"Index": 3, "Name": "Filter", "Type": "TCoLambda"},
+ {"Index": 4, "Name": "Update", "Type": "TCoLambda"}
]
},
{
@@ -98,8 +98,8 @@
"Children": [
{"Index": 0, "Name": "World", "Type": "TExprBase"},
{"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"},
- {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
- {"Index": 3, "Name": "Filter", "Type": "TCoLambda"}
+ {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
+ {"Index": 3, "Name": "Filter", "Type": "TCoLambda"}
]
},
{
@@ -109,9 +109,9 @@
"Children": [
{"Index": 0, "Name": "World", "Type": "TExprBase"},
{"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"},
- {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
{"Index": 3, "Name": "Columns", "Type": "TExprList"},
- {"Index": 4, "Name": "PrimaryKey", "Type": "TCoAtomList"},
+ {"Index": 4, "Name": "PrimaryKey", "Type": "TCoAtomList"},
{"Index": 5, "Name": "Settings", "Type": "TCoNameValueTupleList"},
{"Index": 6, "Name": "Indexes", "Type": "TCoIndexList"},
{"Index": 7, "Name": "PartitionBy", "Type": "TCoAtomList"},
@@ -127,7 +127,7 @@
"Children": [
{"Index": 0, "Name": "World", "Type": "TExprBase"},
{"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"},
- {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
{"Index": 3, "Name": "Actions", "Type": "TCoNameValueTupleList"}
]
},
@@ -138,8 +138,8 @@
"Children": [
{"Index": 0, "Name": "World", "Type": "TExprBase"},
{"Index": 1, "Name": "DataSink", "Type": "TKiDataSink"},
- {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
- {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}
+ {"Index": 2, "Name": "Table", "Type": "TCoAtom"},
+ {"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}
]
},
{
@@ -269,7 +269,7 @@
"Match": {"Type": "NodeType", "TypeName": "List"},
"Builder": {"Generate": "Custom"},
"Children": [
- {"Index": 0, "Name": "Column", "Type": "TCoAtom"},
+ {"Index": 0, "Name": "Column", "Type": "TCoAtom"},
{"Index": 1, "Name": "From", "Type": "TExprBase"},
{"Index": 2, "Name": "To", "Type": "TExprBase"}
]
@@ -279,10 +279,10 @@
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "KiSelectRow"},
"Children": [
- {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"},
+ {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"},
{"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"},
- {"Index": 2, "Name": "Key", "Type": "TCoNameValueTupleList"},
- {"Index": 3, "Name": "Select", "Type": "TCoAtomList"}
+ {"Index": 2, "Name": "Key", "Type": "TCoNameValueTupleList"},
+ {"Index": 3, "Name": "Select", "Type": "TCoAtomList"}
]
},
{
@@ -290,9 +290,9 @@
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "KiUpdateRow"},
"Children": [
- {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"},
+ {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"},
{"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"},
- {"Index": 2, "Name": "Key", "Type": "TCoNameValueTupleList"},
+ {"Index": 2, "Name": "Key", "Type": "TCoNameValueTupleList"},
{"Index": 3, "Name": "Update", "Type": "TCoNameValueTupleList"}
]
},
@@ -301,9 +301,9 @@
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "KiEraseRow"},
"Children": [
- {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"},
+ {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"},
{"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"},
- {"Index": 2, "Name": "Key", "Type": "TCoNameValueTupleList"}
+ {"Index": 2, "Name": "Key", "Type": "TCoNameValueTupleList"}
]
},
{
@@ -312,11 +312,11 @@
"Match": {"Type": "CallableBase"},
"Builder": {"Generate": "None"},
"Children": [
- {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"},
+ {"Index": 0, "Name": "Cluster", "Type": "TCoAtom"},
{"Index": 1, "Name": "Table", "Type": "TKiVersionedTable"},
{"Index": 2, "Name": "Range", "Type": "TExprList"},
- {"Index": 3, "Name": "Select", "Type": "TCoAtomList"},
- {"Index": 4, "Name": "Settings", "Type": "TCoNameValueTupleList"}
+ {"Index": 3, "Name": "Select", "Type": "TCoAtomList"},
+ {"Index": 4, "Name": "Settings", "Type": "TCoNameValueTupleList"}
]
},
{
@@ -337,7 +337,7 @@
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "KiSetResult"},
"Children": [
- {"Index": 0, "Name": "Name", "Type": "TCoAtom"},
+ {"Index": 0, "Name": "Name", "Type": "TCoAtom"},
{"Index": 1, "Name": "Data", "Type": "TExprBase"}
]
},
@@ -355,7 +355,7 @@
"Match": {"Type": "Callable", "Name": "KiMapParameter"},
"Children": [
{"Index": 0, "Name": "Input", "Type": "TExprBase"},
- {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"}
+ {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"}
]
},
{
@@ -373,7 +373,7 @@
"Match": {"Type": "Callable", "Name": "KiFlatMapParameter"},
"Children": [
{"Index": 0, "Name": "Input", "Type": "TExprBase"},
- {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"}
+ {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"}
]
},
{
@@ -384,7 +384,7 @@
"Children": [
{"Index": 0, "Name": "Predicate", "Type": "TExprBase"},
{"Index": 1, "Name": "Effect", "Type": "TExprBase"},
- {"Index": 2, "Name": "Constraint", "Type": "TCoAtom"}
+ {"Index": 2, "Name": "Constraint", "Type": "TCoAtom"}
]
},
{
@@ -413,7 +413,7 @@
"Children": [
{"Index": 0, "Name": "Input", "Type": "TExprBase"},
{"Index": 1, "Name": "SortDirections", "Type": "TExprBase"},
- {"Index": 2, "Name": "KeySelectorLambda", "Type": "TCoLambda"}
+ {"Index": 2, "Name": "KeySelectorLambda", "Type": "TCoLambda"}
]
},
{
diff --git a/ydb/core/kqp/provider/yql_kikimr_mkql.cpp b/ydb/core/kqp/provider/yql_kikimr_mkql.cpp
index 4cd98892e4..95aa0298bf 100644
--- a/ydb/core/kqp/provider/yql_kikimr_mkql.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_mkql.cpp
@@ -166,7 +166,7 @@ TMaybeNode<TExprBase> TranslateToMkql(TExprBase node, TExprContext& ctx, const T
bool hasResult = !program.Results().Empty();
if (hasResult) {
- node = Build<TCoAppend>(ctx, node.Pos())
+ node = Build<TCoAppend>(ctx, node.Pos())
.List(program.Effects())
.Item<TKiSetResult>()
.Name().Build("Result")
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt.cpp b/ydb/core/kqp/provider/yql_kikimr_opt.cpp
index cdbb6e600d..2ffe412837 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_opt.cpp
@@ -16,7 +16,7 @@ bool CanPushPartialSort(const TKiPartialSort& node, const TKikimrTableDescriptio
}
TExprNode::TPtr KiTrimReadTableWorld(TExprBase node) {
- if (auto maybeRead = node.Maybe<TCoLeft>().Input().Maybe<TKiReadTable>()) {
+ if (auto maybeRead = node.Maybe<TCoLeft>().Input().Maybe<TKiReadTable>()) {
YQL_CLOG(INFO, ProviderKikimr) << "KiTrimReadTableWorld";
return maybeRead.Cast().World().Ptr();
}
@@ -25,16 +25,16 @@ TExprNode::TPtr KiTrimReadTableWorld(TExprBase node) {
}
TExprNode::TPtr KiEmptyCommit(TExprBase node) {
- if (!node.Maybe<TCoCommit>().World().Maybe<TCoCommit>()) {
+ if (!node.Maybe<TCoCommit>().World().Maybe<TCoCommit>()) {
return node.Ptr();
}
- auto commit = node.Cast<TCoCommit>();
+ auto commit = node.Cast<TCoCommit>();
if (!commit.DataSink().Maybe<TKiDataSink>()) {
return node.Ptr();
}
- auto innerCommit = commit.World().Cast<TCoCommit>();
+ auto innerCommit = commit.World().Cast<TCoCommit>();
if (!innerCommit.DataSink().Maybe<TKiDataSink>()) {
return node.Ptr();
}
@@ -43,13 +43,13 @@ TExprNode::TPtr KiEmptyCommit(TExprBase node) {
}
TExprNode::TPtr KiEraseOverSelectRow(TExprBase node, TExprContext& ctx) {
- if (!node.Maybe<TCoFlatMap>().Input().Maybe<TKiSelectRow>()) {
+ if (!node.Maybe<TCoFlatMap>().Input().Maybe<TKiSelectRow>()) {
return node.Ptr();
}
- auto map = node.Cast<TCoFlatMap>();
+ auto map = node.Cast<TCoFlatMap>();
- if (auto maybeErase = map.Lambda().Body().Maybe<TCoJust>().Input().Maybe<TKiEraseRow>()) {
+ if (auto maybeErase = map.Lambda().Body().Maybe<TCoJust>().Input().Maybe<TKiEraseRow>()) {
auto selectRow = map.Input().Cast<TKiSelectRow>();
auto eraseRow = maybeErase.Cast();
@@ -101,12 +101,12 @@ TExprNode::TPtr KiEraseOverSelectRow(TExprBase node, TExprContext& ctx) {
}
TExprNode::TPtr KiRewriteAggregate(TExprBase node, TExprContext& ctx) {
- if (!node.Maybe<TCoAggregate>()) {
+ if (!node.Maybe<TCoAggregate>()) {
return node.Ptr();
}
- auto agg = node.Cast<TCoAggregate>();
- if (!agg.Input().Maybe<TKiSelectRange>() && !agg.Input().Maybe<TCoFlatMap>().Input().Maybe<TKiSelectRange>()) {
+ auto agg = node.Cast<TCoAggregate>();
+ if (!agg.Input().Maybe<TKiSelectRange>() && !agg.Input().Maybe<TCoFlatMap>().Input().Maybe<TKiSelectRange>()) {
return node.Ptr();
}
@@ -126,7 +126,7 @@ TExprNode::TPtr KiRewriteAggregate(TExprBase node, TExprContext& ctx) {
TExprNode::TPtr KiRedundantSortByPk(TExprBase node, TExprContext& ctx,
const TKikimrTablesData& tablesData, const TKikimrConfiguration& config)
{
- auto maybeSort = node.Maybe<TCoSort>();
+ auto maybeSort = node.Maybe<TCoSort>();
auto maybePartialSort = node.Maybe<TKiPartialSort>();
if (!maybeSort && !maybePartialSort) {
@@ -139,8 +139,8 @@ TExprNode::TPtr KiRedundantSortByPk(TExprBase node, TExprContext& ctx,
auto read = input;
TMaybe<THashSet<TStringBuf>> passthroughFields;
- if (maybePartialSort && input.Maybe<TCoFlatMap>()) {
- auto flatmap = input.Cast<TCoFlatMap>();
+ if (maybePartialSort && input.Maybe<TCoFlatMap>()) {
+ auto flatmap = input.Cast<TCoFlatMap>();
if (!IsPassthroughFlatMap(flatmap, &passthroughFields)) {
return node.Ptr();
@@ -169,11 +169,11 @@ TExprNode::TPtr KiRedundantSortByPk(TExprBase node, TExprContext& ctx,
};
auto getDirection = [] (TExprBase expr) -> ui32 {
- if (!expr.Maybe<TCoBool>()) {
+ if (!expr.Maybe<TCoBool>()) {
return SortDirectionUnknown;
}
- if (!FromString<bool>(expr.Cast<TCoBool>().Literal().Value())) {
+ if (!FromString<bool>(expr.Cast<TCoBool>().Literal().Value())) {
return SortDirectionReverse;
}
@@ -199,11 +199,11 @@ TExprNode::TPtr KiRedundantSortByPk(TExprBase node, TExprContext& ctx,
auto& tableData = tablesData.ExistingTable(selectRange.Cluster().StringValue(), selectRange.Table().Path().StringValue());
auto checkKey = [keySelector, &tableData, &passthroughFields] (TExprBase key, ui32 index) {
- if (!key.Maybe<TCoMember>()) {
+ if (!key.Maybe<TCoMember>()) {
return false;
}
- auto member = key.Cast<TCoMember>();
+ auto member = key.Cast<TCoMember>();
if (member.Struct().Raw() != keySelector.Args().Arg(0).Raw()) {
return false;
}
@@ -290,7 +290,7 @@ TExprNode::TPtr KiTopSort(TExprBase node, TExprContext& ctx, const TOptimizeCont
return node.Ptr();
}
- auto take = node.Cast<TCoTake>();
+ auto take = node.Cast<TCoTake>();
TCoInputBase top = take;
if (!optCtx.IsSingleUsage(take.Input().Ref())) {
return node.Ptr();
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp
index 0ba0b34952..ea9645ed43 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_opt_build.cpp
@@ -348,7 +348,7 @@ bool IsKikimrPureNode(const TExprNode::TPtr& node) {
return true;
}
- if (!node->GetTypeAnn()->IsComposable()) {
+ if (!node->GetTypeAnn()->IsComposable()) {
return false;
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp
index 662082c543..04e3587ff8 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_opt_join.cpp
@@ -902,7 +902,7 @@ TExprNode::TPtr KiRewriteEquiJoin(TExprBase node, const TKikimrTablesData& table
auto input = join.Arg(i).Cast<TCoEquiJoinInput>();
joinInputs.push_back(input);
- auto itemType = input.List().Ptr()->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ auto itemType = input.List().Ptr()->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
joinLabels.Add(ctx, *input.Scope().Ptr(), itemType->Cast<TStructExprType>());
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp b/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp
index cce81efc42..4b51f2857c 100644
--- a/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_opt_range.cpp
@@ -244,7 +244,7 @@ TMaybeNode<TExprBase> KiTableLookupGetValue(TExprBase node, const TTypeAnnotatio
return ret;
}
- auto valueType = valueNode.Ref().GetTypeAnn();
+ auto valueType = valueNode.Ref().GetTypeAnn();
if (isTargetOptional && valueType->GetKind() == ETypeAnnotationKind::Optional) {
valueType = valueType->Cast<TOptionalExprType>()->GetItemType();
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_provider.cpp b/ydb/core/kqp/provider/yql_kikimr_provider.cpp
index 635d164827..d173ce1ca2 100644
--- a/ydb/core/kqp/provider/yql_kikimr_provider.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_provider.cpp
@@ -400,7 +400,7 @@ NNodes::TCoAtomList BuildColumnsList(
) {
TVector<TExprBase> columnsToSelect;
for (const auto& pair : table.Metadata->Columns) {
- auto atom = Build<TCoAtom>(ctx, pos)
+ auto atom = Build<TCoAtom>(ctx, pos)
.Value(pair.second.Name)
.Done();
diff --git a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h
index e1aa3cda2b..6ec79164f1 100644
--- a/ydb/core/kqp/provider/yql_kikimr_provider_impl.h
+++ b/ydb/core/kqp/provider/yql_kikimr_provider_impl.h
@@ -41,7 +41,7 @@ private:
virtual TStatus HandleAlterGroup(NNodes::TKiAlterGroup node, TExprContext& ctx) = 0;
virtual TStatus HandleDropGroup(NNodes::TKiDropGroup node, TExprContext& ctx) = 0;
virtual TStatus HandleWrite(NNodes::TExprBase node, TExprContext& ctx) = 0;
- virtual TStatus HandleCommit(NNodes::TCoCommit node, TExprContext& ctx) = 0;
+ virtual TStatus HandleCommit(NNodes::TCoCommit node, TExprContext& ctx) = 0;
virtual TStatus HandleKql(NNodes::TCallable node, TExprContext& ctx) = 0;
virtual TStatus HandleExecDataQuery(NNodes::TKiExecDataQuery node, TExprContext& ctx) = 0;
virtual TStatus HandleDataQuery(NNodes::TKiDataQuery node, TExprContext& ctx) = 0;
@@ -220,7 +220,7 @@ NNodes::TCoAtomList BuildKeyColumnsList(
NNodes::TCoAtomList MergeColumns(const NNodes::TCoAtomList& col1, const TVector<TString>& col2, TExprContext& ctx);
bool IsKqlPureExpr(NNodes::TExprBase expr);
-bool IsKqlPureLambda(NNodes::TCoLambda lambda);
+bool IsKqlPureLambda(NNodes::TCoLambda lambda);
bool IsKeySelectorPkPrefix(NNodes::TCoLambda lambda, const TKikimrTableDescription& desc, TVector<TString>* columns);
NNodes::TCoNameValueTupleList ExtractNamedKeyTuples(NNodes::TCoArgument arg,
diff --git a/ydb/core/kqp/provider/yql_kikimr_results.cpp b/ydb/core/kqp/provider/yql_kikimr_results.cpp
index 8ec93f416d..26abefe4bc 100644
--- a/ydb/core/kqp/provider/yql_kikimr_results.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_results.cpp
@@ -960,7 +960,7 @@ void TransformerStatsToProto(const TString& name, const IGraphTransformer::TStat
if (stats.NewTypeNodes > 0) {
profile.SetNewTypeNodes(stats.NewTypeNodes);
}
- // TODO: stats.NewConstraintNodes
+ // TODO: stats.NewConstraintNodes
if (stats.Repeats > 0) {
profile.SetRepeats(stats.Repeats);
}
diff --git a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp
index 692165204a..7be25f29bb 100644
--- a/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp
+++ b/ydb/core/kqp/provider/yql_kikimr_type_ann.cpp
@@ -50,18 +50,18 @@ const TTypeAnnotationNode* GetExpectedRowType(const TKikimrTableDescription& tab
IGraphTransformer::TStatus ConvertTableRowType(TExprNode::TPtr& input, const TKikimrTableDescription& tableDesc,
TExprContext& ctx)
{
- YQL_ENSURE(input->GetTypeAnn());
+ YQL_ENSURE(input->GetTypeAnn());
const TTypeAnnotationNode* actualType;
- switch (input->GetTypeAnn()->GetKind()) {
+ switch (input->GetTypeAnn()->GetKind()) {
case ETypeAnnotationKind::List:
- actualType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ actualType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
break;
case ETypeAnnotationKind::Stream:
- actualType = input->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
+ actualType = input->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
break;
default:
- actualType = input->GetTypeAnn();
+ actualType = input->GetTypeAnn();
break;
}
@@ -74,7 +74,7 @@ IGraphTransformer::TStatus ConvertTableRowType(TExprNode::TPtr& input, const TKi
return IGraphTransformer::TStatus::Error;
}
- switch (input->GetTypeAnn()->GetKind()) {
+ switch (input->GetTypeAnn()->GetKind()) {
case ETypeAnnotationKind::List:
expectedType = ctx.MakeType<TListExprType>(expectedType);
break;
@@ -158,10 +158,10 @@ private:
auto listSelectType = ctx.MakeType<TListExprType>(selectType);
TTypeAnnotationNode::TListType children;
- children.push_back(node.World().Ref().GetTypeAnn());
+ children.push_back(node.World().Ref().GetTypeAnn());
children.push_back(listSelectType);
auto tupleAnn = ctx.MakeType<TTupleExprType>(children);
- node.Ptr()->SetTypeAnn(tupleAnn);
+ node.Ptr()->SetTypeAnn(tupleAnn);
YQL_ENSURE(tableDesc->Metadata->ColumnOrder.size() == tableDesc->Metadata->Columns.size());
return Types.SetColumnOrder(node.Ref(), tableDesc->Metadata->ColumnOrder, ctx);
@@ -171,9 +171,9 @@ private:
{
auto tableListAnnotation = BuildCommonTableListType(ctx);
TTypeAnnotationNode::TListType children;
- children.push_back(node.World().Ref().GetTypeAnn());
+ children.push_back(node.World().Ref().GetTypeAnn());
children.push_back(tableListAnnotation);
- node.Ptr()->SetTypeAnn(ctx.MakeType<TTupleExprType>(children));
+ node.Ptr()->SetTypeAnn(ctx.MakeType<TTupleExprType>(children));
return TStatus::Ok;
}
@@ -185,9 +185,9 @@ private:
}
TTypeAnnotationNode::TListType children;
- children.push_back(node.World().Ref().GetTypeAnn());
+ children.push_back(node.World().Ref().GetTypeAnn());
children.push_back(ctx.MakeType<TDataExprType>(EDataSlot::Yson));
- node.Ptr()->SetTypeAnn(ctx.MakeType<TTupleExprType>(children));
+ node.Ptr()->SetTypeAnn(ctx.MakeType<TTupleExprType>(children));
return TStatus::Ok;
}
@@ -216,7 +216,7 @@ private:
return TStatus::Error;
}
- node.Ptr()->SetTypeAnn(node.Ref().Child(0)->GetTypeAnn());
+ node.Ptr()->SetTypeAnn(node.Ref().Child(0)->GetTypeAnn());
return TStatus::Ok;
}
@@ -278,7 +278,7 @@ private:
return TStatus::Error;
}
- node.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>());
+ node.Ptr()->SetTypeAnn(ctx.MakeType<TUnitExprType>());
return TStatus::Ok;
}
@@ -308,7 +308,7 @@ private:
TVector<TExprBase> convertedValues;
for (const auto& value : tuple) {
- auto valueType = value.Ref().GetTypeAnn();
+ auto valueType = value.Ref().GetTypeAnn();
if (valueType->GetKind() != ETypeAnnotationKind::Struct) {
ctx.AddError(TIssue(pos, TStringBuilder()
<< "Expected structs as input, but got: " << *valueType));
@@ -330,7 +330,7 @@ private:
convertedValues.push_back(TExprBase(node));
}
- auto list = Build<TCoAsList>(ctx, node.Pos())
+ auto list = Build<TCoAsList>(ctx, node.Pos())
.Add(convertedValues)
.Done();
@@ -340,7 +340,7 @@ private:
const TStructExprType* rowType = nullptr;
- auto inputType = node.Input().Ref().GetTypeAnn();
+ auto inputType = node.Input().Ref().GetTypeAnn();
if (inputType->GetKind() == ETypeAnnotationKind::List) {
auto listType = inputType->Cast<TListExprType>();
auto itemType = listType->GetItemType();
@@ -407,11 +407,11 @@ private:
columns.push_back(ctx.NewAtom(node.Pos(), item->GetName()));
}
- node.Ptr()->ChildRef(TKiWriteTable::idx_Settings) = Build<TCoNameValueTupleList>(ctx, node.Pos())
+ node.Ptr()->ChildRef(TKiWriteTable::idx_Settings) = Build<TCoNameValueTupleList>(ctx, node.Pos())
.Add(node.Settings())
.Add()
.Name().Build("input_columns")
- .Value<TCoAtomList>()
+ .Value<TCoAtomList>()
.Add(columns)
.Build()
.Build()
@@ -420,7 +420,7 @@ private:
return TStatus::Repeat;
} else {
- for (const auto& atom : TCoNameValueTuple(inputColumns).Value().Cast<TCoAtomList>()) {
+ for (const auto& atom : TCoNameValueTuple(inputColumns).Value().Cast<TCoAtomList>()) {
YQL_ENSURE(rowType->FindItem(atom.Value()));
}
}
@@ -434,7 +434,7 @@ private:
return TStatus::Error;
}
- node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
+ node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
return TStatus::Ok;
}
@@ -454,7 +454,7 @@ private:
return IGraphTransformer::TStatus::Error;
}
- if (!filterLambda->GetTypeAnn()) {
+ if (!filterLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -467,7 +467,7 @@ private:
return IGraphTransformer::TStatus::Error;
}
- if (!updateLambda->GetTypeAnn()) {
+ if (!updateLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -475,7 +475,7 @@ private:
return IGraphTransformer::TStatus::Error;
}
- auto updateResultType = updateLambda->GetTypeAnn()->Cast<TStructExprType>();
+ auto updateResultType = updateLambda->GetTypeAnn()->Cast<TStructExprType>();
for (auto& item : updateResultType->GetItems()) {
const auto& name = item->GetName();
@@ -500,7 +500,7 @@ private:
auto status = ConvertTableRowType(updateBody, *table, ctx);
if (status != IGraphTransformer::TStatus::Ok) {
if (status == IGraphTransformer::TStatus::Repeat) {
- updateLambda = Build<TCoLambda>(ctx, node.Update().Pos())
+ updateLambda = Build<TCoLambda>(ctx, node.Update().Pos())
.Args(node.Update().Args())
.Body(updateBody)
.Done()
@@ -514,7 +514,7 @@ private:
return TStatus::Error;
}
- node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
+ node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
return TStatus::Ok;
}
@@ -534,7 +534,7 @@ private:
return IGraphTransformer::TStatus::Error;
}
- if (!filterLambda->GetTypeAnn()) {
+ if (!filterLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -546,7 +546,7 @@ private:
return TStatus::Error;
}
- node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
+ node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
return TStatus::Ok;
}
@@ -570,11 +570,11 @@ private:
for (auto item : create.Columns()) {
auto columnTuple = item.Cast<TExprList>();
- auto nameNode = columnTuple.Item(0).Cast<TCoAtom>();
+ auto nameNode = columnTuple.Item(0).Cast<TCoAtom>();
auto typeNode = columnTuple.Item(1);
auto columnName = TString(nameNode.Value());
- auto columnType = typeNode.Ref().GetTypeAnn();
+ auto columnType = typeNode.Ref().GetTypeAnn();
YQL_ENSURE(columnType && columnType->GetKind() == ETypeAnnotationKind::Type);
auto type = columnType->Cast<TTypeExprType>()->GetType();
@@ -832,7 +832,7 @@ private:
bool sysColumnsEnabled = SessionCtx->Config().SystemColumnsEnabled();
YQL_ENSURE(tableDesc.Load(ctx, sysColumnsEnabled));
- create.Ptr()->SetTypeAnn(create.World().Ref().GetTypeAnn());
+ create.Ptr()->SetTypeAnn(create.World().Ref().GetTypeAnn());
return TStatus::Ok;
}
@@ -850,7 +850,7 @@ private:
return TStatus::Error;
}
- node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
+ node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
return TStatus::Ok;
}
@@ -1018,7 +1018,7 @@ private:
}
}
- node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
+ node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
return TStatus::Ok;
}
@@ -1133,7 +1133,7 @@ private:
return TStatus::Error;
}
- virtual TStatus HandleCommit(NNodes::TCoCommit node, TExprContext& ctx) override {
+ virtual TStatus HandleCommit(NNodes::TCoCommit node, TExprContext& ctx) override {
auto settings = NCommon::ParseCommitSettings(node, ctx);
bool isFlushCommit = false;
@@ -1172,7 +1172,7 @@ private:
break;
}
- node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
+ node.Ptr()->SetTypeAnn(node.World().Ref().GetTypeAnn());
return TStatus::Ok;
}
@@ -1235,10 +1235,10 @@ private:
}
TTypeAnnotationNode::TListType children;
- children.push_back(node.World().Ref().GetTypeAnn());
+ children.push_back(node.World().Ref().GetTypeAnn());
children.push_back(node.Query().Ref().GetTypeAnn());
auto tupleAnn = ctx.MakeType<TTupleExprType>(children);
- node.Ptr()->SetTypeAnn(tupleAnn);
+ node.Ptr()->SetTypeAnn(tupleAnn);
return TStatus::Ok;
}
@@ -1256,7 +1256,7 @@ private:
auto optSelectType = ctx.MakeType<TOptionalExprType>(selectType);
- node.Ptr()->SetTypeAnn(optSelectType);
+ node.Ptr()->SetTypeAnn(optSelectType);
return TStatus::Ok;
}
@@ -1278,19 +1278,19 @@ private:
}
if (node.Maybe<TKiUpdateRow>()) {
- node.Ptr()->SetTypeAnn(ctx.MakeType<TVoidExprType>());
+ node.Ptr()->SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
if (node.Maybe<TKiEraseRow>()) {
- node.Ptr()->SetTypeAnn(ctx.MakeType<TVoidExprType>());
+ node.Ptr()->SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
if (node.Maybe<TKiSetResult>()) {
- node.Ptr()->SetTypeAnn(ctx.MakeType<TVoidExprType>());
+ node.Ptr()->SetTypeAnn(ctx.MakeType<TVoidExprType>());
return TStatus::Ok;
}
@@ -1307,16 +1307,16 @@ private:
}
auto& lambda = map.Ptr()->ChildRef(TKiMapParameter::idx_Lambda);
- auto itemType = map.Input().Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ auto itemType = map.Input().Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
if (!UpdateLambdaAllArgumentsTypes(lambda, {itemType}, ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- map.Ptr()->SetTypeAnn(ctx.MakeType<TListExprType>(lambda->GetTypeAnn()));
+ map.Ptr()->SetTypeAnn(ctx.MakeType<TListExprType>(lambda->GetTypeAnn()));
return TStatus::Ok;
}
@@ -1333,22 +1333,22 @@ private:
}
auto& lambda = map.Ptr()->ChildRef(TKiFlatMapParameter::idx_Lambda);
- auto itemType = map.Input().Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ auto itemType = map.Input().Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
if (!UpdateLambdaAllArgumentsTypes(lambda, {itemType}, ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto retKind = lambda->GetTypeAnn()->GetKind();
+ auto retKind = lambda->GetTypeAnn()->GetKind();
if (retKind != ETypeAnnotationKind::List) {
ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << "Expected list as labmda return type, but got: " << *lambda->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
- map.Ptr()->SetTypeAnn(lambda->GetTypeAnn());
+ map.Ptr()->SetTypeAnn(lambda->GetTypeAnn());
return TStatus::Ok;
}
@@ -1372,7 +1372,7 @@ private:
return IGraphTransformer::TStatus::Error;
}
- auto predicateType = condEffect.Predicate().Ref().GetTypeAnn()->Cast<TDataExprType>();
+ auto predicateType = condEffect.Predicate().Ref().GetTypeAnn()->Cast<TDataExprType>();
YQL_ENSURE(predicateType);
if (predicateType->GetSlot() != EDataSlot::Bool) {
@@ -1384,7 +1384,7 @@ private:
return IGraphTransformer::TStatus::Error;
}
- condEffect.Ptr()->SetTypeAnn(condEffect.Effect().Ref().GetTypeAnn());
+ condEffect.Ptr()->SetTypeAnn(condEffect.Effect().Ref().GetTypeAnn());
return TStatus::Ok;
}
diff --git a/ydb/core/kqp/ut/kqp_join_ut.cpp b/ydb/core/kqp/ut/kqp_join_ut.cpp
index 8fa59d31b3..e8ce3b93ae 100644
--- a/ydb/core/kqp/ut/kqp_join_ut.cpp
+++ b/ydb/core/kqp/ut/kqp_join_ut.cpp
@@ -878,8 +878,8 @@ Y_UNIT_TEST_SUITE(KqpJoin) {
[[101];[1];["Value1"];[101];["Value21"]];
[[101];[1];["Value1"];[101];["Value22"]];
[[101];[1];["Value1"];[101];["Value23"]];
- [[102];[2];["Value1"];[102];["Value24"]];
- [[103];[3];["Value2"];[103];["Value25"]];
+ [[102];[2];["Value1"];[102];["Value24"]];
+ [[103];[3];["Value2"];[103];["Value25"]];
[[104];[4];["Value2"];[104];["Value26"]];
[[105];[5];["Value3"];[105];["Value27"]];
[[105];[5];["Value3"];[105];["Value28"]];
diff --git a/ydb/core/scheme_types/scheme_type_traits.h b/ydb/core/scheme_types/scheme_type_traits.h
index 7e07f95976..f8de61365d 100644
--- a/ydb/core/scheme_types/scheme_type_traits.h
+++ b/ydb/core/scheme_types/scheme_type_traits.h
@@ -1,57 +1,57 @@
-#pragma once
-
+#pragma once
+
#include <ydb/public/lib/scheme_types/scheme_type_id.h>
-
-namespace NKikimr {
-namespace NScheme {
-
-constexpr bool IsNumericType(TTypeId id) noexcept {
+
+namespace NKikimr {
+namespace NScheme {
+
+constexpr bool IsNumericType(TTypeId id) noexcept {
return NTypeIds::Int8 == id
|| NTypeIds::Uint8 == id
|| NTypeIds::Int16 == id
|| NTypeIds::Uint16 == id
- || NTypeIds::Int32 == id
- || NTypeIds::Uint32 == id
- || NTypeIds::Int64 == id
- || NTypeIds::Uint64 == id
- || NTypeIds::Double == id
- || NTypeIds::Float == id;
-}
-
-constexpr bool IsIntegralType(TTypeId id) noexcept {
+ || NTypeIds::Int32 == id
+ || NTypeIds::Uint32 == id
+ || NTypeIds::Int64 == id
+ || NTypeIds::Uint64 == id
+ || NTypeIds::Double == id
+ || NTypeIds::Float == id;
+}
+
+constexpr bool IsIntegralType(TTypeId id) noexcept {
return NTypeIds::Int8 == id
|| NTypeIds::Uint8 == id
|| NTypeIds::Int16 == id
|| NTypeIds::Uint16 == id
- || NTypeIds::Int32 == id
- || NTypeIds::Uint32 == id
- || NTypeIds::Int64 == id
- || NTypeIds::Uint64 == id;
-}
-
-constexpr bool IsSignedIntegerType(TTypeId id) noexcept {
+ || NTypeIds::Int32 == id
+ || NTypeIds::Uint32 == id
+ || NTypeIds::Int64 == id
+ || NTypeIds::Uint64 == id;
+}
+
+constexpr bool IsSignedIntegerType(TTypeId id) noexcept {
return NTypeIds::Int8 == id || NTypeIds::Int16 == id || NTypeIds::Int32 == id || NTypeIds::Int64 == id;
-}
-
-constexpr bool IsUnsignedIntegerType(TTypeId id) noexcept {
+}
+
+constexpr bool IsUnsignedIntegerType(TTypeId id) noexcept {
return NTypeIds::Uint8 == id || NTypeIds::Uint16 == id || NTypeIds::Uint32 == id || NTypeIds::Uint64 == id;
-}
-
-constexpr bool IsFloatingType(TTypeId id) noexcept {
- return NTypeIds::Double == id || NTypeIds::Float == id;
-}
-
-constexpr bool IsStringType(TTypeId id) noexcept {
+}
+
+constexpr bool IsFloatingType(TTypeId id) noexcept {
+ return NTypeIds::Double == id || NTypeIds::Float == id;
+}
+
+constexpr bool IsStringType(TTypeId id) noexcept {
return NTypeIds::String == id
|| NTypeIds::String4k == id
|| NTypeIds::String2m == id
|| NTypeIds::Utf8 == id
- || NTypeIds::Yson == id
+ || NTypeIds::Yson == id
|| NTypeIds::Json == id
|| NTypeIds::JsonDocument == id
|| NTypeIds::DyNumber
- ;
-}
-
-} // namspace NScheme
-} // namspace NKikimr
+ ;
+}
+
+} // namspace NScheme
+} // namspace NKikimr
diff --git a/ydb/core/testlib/minikql_compile.h b/ydb/core/testlib/minikql_compile.h
index 4d76ca8d73..63aa8a049b 100644
--- a/ydb/core/testlib/minikql_compile.h
+++ b/ydb/core/testlib/minikql_compile.h
@@ -82,7 +82,7 @@ inline TExprContainer::TPtr ParseText(const TString& programText) {
UNIT_ASSERT(astRes.IsOk());
TExprContainer::TPtr expr(new TExprContainer());
- bool isOk = CompileExpr(*astRes.Root, expr->Root, expr->Context, nullptr);
+ bool isOk = CompileExpr(*astRes.Root, expr->Root, expr->Context, nullptr);
expr->Context.IssueManager.GetIssues().PrintTo(Cerr);
UNIT_ASSERT(isOk);
return expr;
diff --git a/ydb/core/util/tracking_alloc.h b/ydb/core/util/tracking_alloc.h
index 96855506ac..df57f19811 100644
--- a/ydb/core/util/tracking_alloc.h
+++ b/ydb/core/util/tracking_alloc.h
@@ -1,48 +1,48 @@
-#pragma once
-
-#include <util/system/yassert.h>
-#include <util/generic/ptr.h>
-
-#include <memory>
-
-namespace NKikimr {
-
-template <typename T>
-class TTrackingAllocator : public std::allocator<T> {
-public:
- typedef size_t size_type;
- typedef T* pointer;
- typedef const T* const_pointer;
-
- template<typename U>
- struct rebind {
- typedef TTrackingAllocator<U> other;
- };
-
- struct propagate_on_container_copy_assignment : public std::true_type {};
- struct propagate_on_container_move_assignment : public std::true_type {};
- struct propagate_on_container_swap : public std::true_type {};
-
- template<typename U> friend class TTrackingAllocator;
-
+#pragma once
+
+#include <util/system/yassert.h>
+#include <util/generic/ptr.h>
+
+#include <memory>
+
+namespace NKikimr {
+
+template <typename T>
+class TTrackingAllocator : public std::allocator<T> {
+public:
+ typedef size_t size_type;
+ typedef T* pointer;
+ typedef const T* const_pointer;
+
+ template<typename U>
+ struct rebind {
+ typedef TTrackingAllocator<U> other;
+ };
+
+ struct propagate_on_container_copy_assignment : public std::true_type {};
+ struct propagate_on_container_move_assignment : public std::true_type {};
+ struct propagate_on_container_swap : public std::true_type {};
+
+ template<typename U> friend class TTrackingAllocator;
+
TTrackingAllocator() noexcept
- : std::allocator<T>()
- , Allocated(new size_t(0))
- {
- }
-
+ : std::allocator<T>()
+ , Allocated(new size_t(0))
+ {
+ }
+
TTrackingAllocator(const TSimpleSharedPtr<size_t>& allocated) noexcept
- : std::allocator<T>()
- , Allocated(allocated)
- {
- }
-
+ : std::allocator<T>()
+ , Allocated(allocated)
+ {
+ }
+
TTrackingAllocator(const TTrackingAllocator& a) noexcept
- : std::allocator<T>(a)
- , Allocated(a.Allocated)
- {
- }
-
+ : std::allocator<T>(a)
+ , Allocated(a.Allocated)
+ {
+ }
+
TTrackingAllocator(TTrackingAllocator&& a) noexcept
: std::allocator<T>(std::move(static_cast<std::allocator<T>&&>(a)))
, Allocated(a.Allocated)
@@ -50,13 +50,13 @@ public:
a.Allocated = nullptr;
}
- template <class U>
+ template <class U>
TTrackingAllocator(const TTrackingAllocator<U>& a) noexcept
- : std::allocator<T>(a)
- , Allocated(a.Allocated)
- {
- }
-
+ : std::allocator<T>(a)
+ , Allocated(a.Allocated)
+ {
+ }
+
TTrackingAllocator& operator=(const TTrackingAllocator& a) noexcept
{
Allocated = a.Allocated;
@@ -73,29 +73,29 @@ public:
}
~TTrackingAllocator() {
- }
-
+ }
+
pointer allocate(size_type n, const void* hint = nullptr) {
- *Allocated += n * sizeof(T);
- return std::allocator<T>::allocate(n, hint);
- }
-
- void deallocate(pointer p, size_type n) {
+ *Allocated += n * sizeof(T);
+ return std::allocator<T>::allocate(n, hint);
+ }
+
+ void deallocate(pointer p, size_type n) {
Y_ASSERT(*Allocated >= n * sizeof(T));
- *Allocated -= n * sizeof(T);
- std::allocator<T>::deallocate(p, n);
- }
-
- size_t GetAllocated() const {
- return *Allocated;
- }
-
+ *Allocated -= n * sizeof(T);
+ std::allocator<T>::deallocate(p, n);
+ }
+
+ size_t GetAllocated() const {
+ return *Allocated;
+ }
+
TSimpleSharedPtr<size_t> GetAllocatedPtr() const {
return Allocated;
}
-private:
- TSimpleSharedPtr<size_t> Allocated;
-};
-
-} // NKikimr
+private:
+ TSimpleSharedPtr<size_t> Allocated;
+};
+
+} // NKikimr
diff --git a/ydb/core/yq/libs/gateway/empty_gateway.cpp b/ydb/core/yq/libs/gateway/empty_gateway.cpp
index 01bfd9bfcb..4b231e0d9b 100644
--- a/ydb/core/yq/libs/gateway/empty_gateway.cpp
+++ b/ydb/core/yq/libs/gateway/empty_gateway.cpp
@@ -30,16 +30,16 @@ public:
NYql::NDqs::IDqsExecutionPlanner& plan,
const TVector<TString>& columns,
const THashMap<TString, TString>& secureParams,
- const THashMap<TString, TString>& queryParams,
+ const THashMap<TString, TString>& queryParams,
const NYql::TDqSettings::TPtr& settings,
const TDqProgressWriter& progressWriter,
const THashMap<TString, TString>& modulesMapping,
- bool discard) override
+ bool discard) override
{
Y_UNUSED(progressWriter);
Y_UNUSED(modulesMapping); // TODO: support.
Y_UNUSED(discard);
- Y_UNUSED(queryParams);
+ Y_UNUSED(queryParams);
NProto::TGraphParams params;
THashMap<i64, TString> stagePrograms;
diff --git a/ydb/core/yql_testlib/yql_testlib.cpp b/ydb/core/yql_testlib/yql_testlib.cpp
index 35838c40f6..474b608e32 100644
--- a/ydb/core/yql_testlib/yql_testlib.cpp
+++ b/ydb/core/yql_testlib/yql_testlib.cpp
@@ -191,7 +191,7 @@ void TYqlServer::Initialize() {
void MakeGatewaysConfig(const THashMap<TString, TString>& clusterMapping, NYql::TGatewaysConfig& gatewaysConfig) {
for (auto& x : clusterMapping) {
- if (x.second == NYql::YtProviderName) {
+ if (x.second == NYql::YtProviderName) {
auto cluster = gatewaysConfig.MutableYt()->AddClusterMapping();
cluster->SetName(x.first);
}
diff --git a/ydb/library/yql/ast/ya.make b/ydb/library/yql/ast/ya.make
index 2d55860c89..0e22e0967d 100644
--- a/ydb/library/yql/ast/ya.make
+++ b/ydb/library/yql/ast/ya.make
@@ -10,8 +10,8 @@ OWNER(
SRCS(
yql_ast.cpp
yql_ast.h
- yql_constraint.cpp
- yql_constraint.h
+ yql_constraint.cpp
+ yql_constraint.h
yql_ast_annotation.cpp
yql_ast_annotation.h
yql_ast_escaping.cpp
@@ -32,7 +32,7 @@ SRCS(
PEERDIR(
library/cpp/colorizer
- library/cpp/containers/sorted_vector
+ library/cpp/containers/sorted_vector
library/cpp/containers/stack_vector
library/cpp/deprecated/enum_codegen
library/cpp/enumbitset
diff --git a/ydb/library/yql/ast/yql_ast.h b/ydb/library/yql/ast/yql_ast.h
index 887b53dbc5..5efa5a397f 100644
--- a/ydb/library/yql/ast/yql_ast.h
+++ b/ydb/library/yql/ast/yql_ast.h
@@ -181,7 +181,7 @@ struct TAstNode {
inline ~TAstNode() {}
void Destroy() {
- TString().swap(Position.File);
+ TString().swap(Position.File);
}
private:
diff --git a/ydb/library/yql/ast/yql_ast_ut.cpp b/ydb/library/yql/ast/yql_ast_ut.cpp
index 23e140653d..c51a981b17 100644
--- a/ydb/library/yql/ast/yql_ast_ut.cpp
+++ b/ydb/library/yql/ast/yql_ast_ut.cpp
@@ -83,38 +83,38 @@ Y_UNIT_TEST_SUITE(TParseYqlAst) {
}
Y_UNIT_TEST(AnnotatedAstPrint) {
- TMemoryPool pool(4096);
- TAstParseResult ast = ParseAst(TEST_PROGRAM, &pool);
+ TMemoryPool pool(4096);
+ TAstParseResult ast = ParseAst(TEST_PROGRAM, &pool);
UNIT_ASSERT(ast.IsOk());
- TAstParseResult astWithPositions;
- astWithPositions.Root = AnnotatePositions(*ast.Root, pool);
- UNIT_ASSERT(!!astWithPositions.Root);
+ TAstParseResult astWithPositions;
+ astWithPositions.Root = AnnotatePositions(*ast.Root, pool);
+ UNIT_ASSERT(!!astWithPositions.Root);
- TString sAnn = astWithPositions.Root->ToString();
+ TString sAnn = astWithPositions.Root->ToString();
UNIT_ASSERT(false == sAnn.empty());
TAstParseResult annRes = ParseAst(sAnn);
UNIT_ASSERT(annRes.IsOk());
- TAstParseResult removedAnn;
- removedAnn.Root = RemoveAnnotations(*annRes.Root, pool);
- UNIT_ASSERT(!!removedAnn.Root);
+ TAstParseResult removedAnn;
+ removedAnn.Root = RemoveAnnotations(*annRes.Root, pool);
+ UNIT_ASSERT(!!removedAnn.Root);
TString strOriginal = ast.Root->ToString();
- TString strAnnRemoved = removedAnn.Root->ToString();
+ TString strAnnRemoved = removedAnn.Root->ToString();
UNIT_ASSERT_VALUES_EQUAL(strOriginal, strAnnRemoved);
- astWithPositions.Root->GetChild(0)->SetContent("100:100", pool);
+ astWithPositions.Root->GetChild(0)->SetContent("100:100", pool);
- TAstParseResult appliedPositionsAnn;
- appliedPositionsAnn.Root = ApplyPositionAnnotations(*astWithPositions.Root, 0, pool);
- UNIT_ASSERT(appliedPositionsAnn.Root);
-
- TAstParseResult removedAnn2;
- removedAnn2.Root = RemoveAnnotations(*appliedPositionsAnn.Root, pool);
- UNIT_ASSERT(removedAnn2.Root);
- UNIT_ASSERT_VALUES_EQUAL(removedAnn2.Root->GetPosition().Row, 100);
+ TAstParseResult appliedPositionsAnn;
+ appliedPositionsAnn.Root = ApplyPositionAnnotations(*astWithPositions.Root, 0, pool);
+ UNIT_ASSERT(appliedPositionsAnn.Root);
+
+ TAstParseResult removedAnn2;
+ removedAnn2.Root = RemoveAnnotations(*appliedPositionsAnn.Root, pool);
+ UNIT_ASSERT(removedAnn2.Root);
+ UNIT_ASSERT_VALUES_EQUAL(removedAnn2.Root->GetPosition().Row, 100);
}
template <typename TCharType>
diff --git a/ydb/library/yql/ast/yql_constraint.cpp b/ydb/library/yql/ast/yql_constraint.cpp
index 74adceb153..6ef4908ce4 100644
--- a/ydb/library/yql/ast/yql_constraint.cpp
+++ b/ydb/library/yql/ast/yql_constraint.cpp
@@ -1,66 +1,66 @@
-#include "yql_constraint.h"
-#include "yql_expr.h"
-
-#include <util/digest/murmur.h>
-#include <util/generic/utility.h>
-#include <util/generic/algorithm.h>
+#include "yql_constraint.h"
+#include "yql_expr.h"
+
+#include <util/digest/murmur.h>
+#include <util/generic/utility.h>
+#include <util/generic/algorithm.h>
#include <util/string/join.h>
-
-#include <algorithm>
-#include <iterator>
-
-namespace NYql {
-
+
+#include <algorithm>
+#include <iterator>
+
+namespace NYql {
+
TConstraintNode::TConstraintNode(TExprContext& ctx, std::string_view name)
- : Hash_(MurmurHash<ui64>(name.data(), name.size()))
- , Name_(ctx.AppendString(name))
-{
-}
-
-TConstraintNode::TConstraintNode(TConstraintNode&& constr)
- : Hash_(constr.Hash_)
- , Name_(constr.Name_)
-{
- constr.Hash_ = 0;
- constr.Name_ = {};
-}
-
-void TConstraintNode::Out(IOutputStream& out) const {
- out.Write(Name_);
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
+ : Hash_(MurmurHash<ui64>(name.data(), name.size()))
+ , Name_(ctx.AppendString(name))
+{
+}
+
+TConstraintNode::TConstraintNode(TConstraintNode&& constr)
+ : Hash_(constr.Hash_)
+ , Name_(constr.Name_)
+{
+ constr.Hash_ = 0;
+ constr.Name_ = {};
+}
+
+void TConstraintNode::Out(IOutputStream& out) const {
+ out.Write(Name_);
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
const TConstraintNode* TConstraintSet::GetConstraint(std::string_view name) const {
const auto it = std::lower_bound(Constraints_.cbegin(), Constraints_.cend(), name, TConstraintNode::TCompare());
- if (it != Constraints_.cend() && (*it)->GetName() == name) {
- return *it;
- }
- return nullptr;
-}
-
-void TConstraintSet::AddConstraint(const TConstraintNode* node) {
- if (!node) {
- return;
- }
+ if (it != Constraints_.cend() && (*it)->GetName() == name) {
+ return *it;
+ }
+ return nullptr;
+}
+
+void TConstraintSet::AddConstraint(const TConstraintNode* node) {
+ if (!node) {
+ return;
+ }
const auto it = std::lower_bound(Constraints_.begin(), Constraints_.end(), node, TConstraintNode::TCompare());
- if (it == Constraints_.end() || (*it)->GetName() != node->GetName()) {
- Constraints_.insert(it, node);
- } else {
- Y_ENSURE(node->Equals(**it), "Adding unequal constraint: " << *node << " != " << **it);
- }
-}
-
+ if (it == Constraints_.end() || (*it)->GetName() != node->GetName()) {
+ Constraints_.insert(it, node);
+ } else {
+ Y_ENSURE(node->Equals(**it), "Adding unequal constraint: " << *node << " != " << **it);
+ }
+}
+
const TConstraintNode* TConstraintSet::RemoveConstraint(std::string_view name) {
- const TConstraintNode* res = nullptr;
+ const TConstraintNode* res = nullptr;
const auto it = std::lower_bound(Constraints_.begin(), Constraints_.end(), name, TConstraintNode::TCompare());
- if (it != Constraints_.end() && (*it)->GetName() == name) {
- res = *it;
- Constraints_.erase(it);
- }
- return res;
-}
-
+ if (it != Constraints_.end() && (*it)->GetName() == name) {
+ res = *it;
+ Constraints_.erase(it);
+ }
+ return res;
+}
+
void TConstraintSet::ToJson(NJson::TJsonWriter& writer) const {
writer.OpenMap();
for (const auto& node : Constraints_) {
@@ -70,97 +70,97 @@ void TConstraintSet::ToJson(NJson::TJsonWriter& writer) const {
writer.CloseMap();
}
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-TColumnSetConstraintNodeBase::TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TSetType& columns)
- : TConstraintNode(ctx, name)
-{
- YQL_ENSURE(!columns.empty());
- for (auto& c: columns) {
- Hash_ = MurmurHash<ui64>(c.data(), c.size(), Hash_);
- Columns_.push_back(ctx.AppendString(c));
- }
-}
-
-TColumnSetConstraintNodeBase::TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TVector<TStringBuf>& columns)
- : TConstraintNode(ctx, name)
-{
- YQL_ENSURE(!columns.empty());
- for (auto& c: columns) {
- YQL_ENSURE(Columns_.insert_unique(ctx.AppendString(c)).second, "Duplicate Unique constraint column: " << c);
- }
- for (auto& c: Columns_) {
- Hash_ = MurmurHash<ui64>(c.data(), c.size(), Hash_);
- }
-}
-
-TColumnSetConstraintNodeBase::TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TVector<TString>& columns)
- : TConstraintNode(ctx, name)
-{
- YQL_ENSURE(!columns.empty());
- for (auto& c: columns) {
- YQL_ENSURE(Columns_.insert_unique(ctx.AppendString(c)).second, "Duplicate Unique constraint column: " << c);
- }
- for (auto& c: Columns_) {
- Hash_ = MurmurHash<ui64>(c.data(), c.size(), Hash_);
- }
-}
-
-TColumnSetConstraintNodeBase::TColumnSetConstraintNodeBase(TColumnSetConstraintNodeBase&& constr)
- : TConstraintNode(std::move(static_cast<TConstraintNode&>(constr)))
- , Columns_(std::move(constr.Columns_))
-{
-}
-
-bool TColumnSetConstraintNodeBase::Equals(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
- if (GetHash() != node.GetHash()) {
- return false;
- }
- if (GetName() != node.GetName()) {
- return false;
- }
- if (auto c = dynamic_cast<const TColumnSetConstraintNodeBase*>(&node)) {
- return Columns_ == c->Columns_;
- }
- return false;
-}
-
-bool TColumnSetConstraintNodeBase::Includes(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
- if (GetName() != node.GetName()) {
- return false;
- }
- if (auto c = dynamic_cast<const TColumnSetConstraintNodeBase*>(&node)) {
- for (auto& col: c->Columns_) {
- if (!Columns_.has(col)) {
- return false;
- }
- }
- return true;
- }
- return false;
-}
-
-void TColumnSetConstraintNodeBase::Out(IOutputStream& out) const {
- TConstraintNode::Out(out);
- out.Write('(');
-
- bool first = true;
- for (auto& col: Columns_) {
- if (!first) {
- out.Write(',');
- }
- out.Write(col);
- first = false;
- }
- out.Write(')');
-}
-
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TColumnSetConstraintNodeBase::TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TSetType& columns)
+ : TConstraintNode(ctx, name)
+{
+ YQL_ENSURE(!columns.empty());
+ for (auto& c: columns) {
+ Hash_ = MurmurHash<ui64>(c.data(), c.size(), Hash_);
+ Columns_.push_back(ctx.AppendString(c));
+ }
+}
+
+TColumnSetConstraintNodeBase::TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TVector<TStringBuf>& columns)
+ : TConstraintNode(ctx, name)
+{
+ YQL_ENSURE(!columns.empty());
+ for (auto& c: columns) {
+ YQL_ENSURE(Columns_.insert_unique(ctx.AppendString(c)).second, "Duplicate Unique constraint column: " << c);
+ }
+ for (auto& c: Columns_) {
+ Hash_ = MurmurHash<ui64>(c.data(), c.size(), Hash_);
+ }
+}
+
+TColumnSetConstraintNodeBase::TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TVector<TString>& columns)
+ : TConstraintNode(ctx, name)
+{
+ YQL_ENSURE(!columns.empty());
+ for (auto& c: columns) {
+ YQL_ENSURE(Columns_.insert_unique(ctx.AppendString(c)).second, "Duplicate Unique constraint column: " << c);
+ }
+ for (auto& c: Columns_) {
+ Hash_ = MurmurHash<ui64>(c.data(), c.size(), Hash_);
+ }
+}
+
+TColumnSetConstraintNodeBase::TColumnSetConstraintNodeBase(TColumnSetConstraintNodeBase&& constr)
+ : TConstraintNode(std::move(static_cast<TConstraintNode&>(constr)))
+ , Columns_(std::move(constr.Columns_))
+{
+}
+
+bool TColumnSetConstraintNodeBase::Equals(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
+ if (GetHash() != node.GetHash()) {
+ return false;
+ }
+ if (GetName() != node.GetName()) {
+ return false;
+ }
+ if (auto c = dynamic_cast<const TColumnSetConstraintNodeBase*>(&node)) {
+ return Columns_ == c->Columns_;
+ }
+ return false;
+}
+
+bool TColumnSetConstraintNodeBase::Includes(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
+ if (GetName() != node.GetName()) {
+ return false;
+ }
+ if (auto c = dynamic_cast<const TColumnSetConstraintNodeBase*>(&node)) {
+ for (auto& col: c->Columns_) {
+ if (!Columns_.has(col)) {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+}
+
+void TColumnSetConstraintNodeBase::Out(IOutputStream& out) const {
+ TConstraintNode::Out(out);
+ out.Write('(');
+
+ bool first = true;
+ for (auto& col: Columns_) {
+ if (!first) {
+ out.Write(',');
+ }
+ out.Write(col);
+ first = false;
+ }
+ out.Write(')');
+}
+
void TColumnSetConstraintNodeBase::ToJson(NJson::TJsonWriter& out) const {
out.OpenArray();
for (const auto& column : Columns_) {
@@ -169,47 +169,47 @@ void TColumnSetConstraintNodeBase::ToJson(NJson::TJsonWriter& out) const {
out.CloseArray();
}
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
TSortedConstraintNode::TSortedConstraintNode(TExprContext& ctx, TContainerType&& content)
: TConstraintNode(ctx, Name()), Content_(std::move(content))
-{
+{
YQL_ENSURE(!Content_.empty());
for (const auto& c : Content_) {
Hash_ = std::accumulate(c.first.cbegin(), c.first.cend(), c.second ? Hash_ : ~Hash_, [](ui64 hash, const std::string_view& field) { return MurmurHash<ui64>(field.data(), field.size(), hash); });
}
-}
-
-TSortedConstraintNode::TSortedConstraintNode(TExprContext& ctx, const TSortedConstraintNode& constr, size_t prefixLength)
+}
+
+TSortedConstraintNode::TSortedConstraintNode(TExprContext& ctx, const TSortedConstraintNode& constr, size_t prefixLength)
: TConstraintNode(ctx, Name()), Content_(constr.Content_)
-{
+{
YQL_ENSURE(prefixLength > 0U);
Content_.resize(prefixLength);
for (const auto& c : Content_) {
Hash_ = std::accumulate(c.first.cbegin(), c.first.cend(), c.second ? Hash_ : ~Hash_, [](ui64 hash, const std::string_view& field) { return MurmurHash<ui64>(field.data(), field.size(), hash); });
}
-}
-
+}
+
TSortedConstraintNode::TSortedConstraintNode(TSortedConstraintNode&&) = default;
-
-bool TSortedConstraintNode::Equals(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
+
+bool TSortedConstraintNode::Equals(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
if (const auto c = dynamic_cast<const TSortedConstraintNode*>(&node)) {
return GetContent() == c->GetContent();
- }
- return false;
-}
-
-bool TSortedConstraintNode::Includes(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
+ }
+ return false;
+}
+
+bool TSortedConstraintNode::Includes(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
if (GetName() != node.GetName()) {
return false;
- }
+ }
const auto& content = static_cast<const TSortedConstraintNode&>(node).GetContent();
if (content.size() > Content_.size())
@@ -221,23 +221,23 @@ bool TSortedConstraintNode::Includes(const TConstraintNode& node) const {
}
return true;
-}
-
-void TSortedConstraintNode::Out(IOutputStream& out) const {
- TConstraintNode::Out(out);
- out.Write('(');
+}
+
+void TSortedConstraintNode::Out(IOutputStream& out) const {
+ TConstraintNode::Out(out);
+ out.Write('(');
for (size_t i = 0; i < Content_.size(); ++i) {
- if (i > 0) {
+ if (i > 0) {
out.Write(';');
- }
+ }
out.Write(JoinSeq(',', Content_[i].first));
- out.Write('[');
+ out.Write('[');
out.Write(Content_[i].second ? "asc" : "desc");
- out.Write(']');
- }
- out.Write(')');
-}
-
+ out.Write(']');
+ }
+ out.Write(')');
+}
+
void TSortedConstraintNode::ToJson(NJson::TJsonWriter& out) const {
out.OpenArray();
for (size_t i = 0; i < Content_.size(); ++i) {
@@ -249,45 +249,45 @@ void TSortedConstraintNode::ToJson(NJson::TJsonWriter& out) const {
out.CloseArray();
}
-bool TSortedConstraintNode::IsPrefixOf(const TSortedConstraintNode& node) const {
+bool TSortedConstraintNode::IsPrefixOf(const TSortedConstraintNode& node) const {
return node.Includes(*this);
-}
-
-size_t TSortedConstraintNode::GetCommonPrefixLength(const TSortedConstraintNode& node) const {
+}
+
+size_t TSortedConstraintNode::GetCommonPrefixLength(const TSortedConstraintNode& node) const {
const size_t minSize = std::min(Content_.size(), node.Content_.size());
for (size_t i = 0U; i < minSize; ++i) {
if (Content_[i] != node.Content_[i]) {
- return i;
- }
- }
- return minSize;
-}
-
-const TSortedConstraintNode* TSortedConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
- if (constraints.empty()) {
- return nullptr;
- }
-
+ return i;
+ }
+ }
+ return minSize;
+}
+
+const TSortedConstraintNode* TSortedConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
+ if (constraints.empty()) {
+ return nullptr;
+ }
+
const auto sort = constraints.front()->GetConstraint<TSortedConstraintNode>();
- if (constraints.size() == 1 || !sort) {
- return sort;
- }
-
+ if (constraints.size() == 1 || !sort) {
+ return sort;
+ }
+
auto commonPrefixLength = sort->GetContent().size();
- for (size_t i = 1; i < constraints.size() && commonPrefixLength > 0; ++i) {
+ for (size_t i = 1; i < constraints.size() && commonPrefixLength > 0; ++i) {
if (const auto nextSort = constraints[i]->GetConstraint<TSortedConstraintNode>()) {
commonPrefixLength = std::min(commonPrefixLength, nextSort->GetCommonPrefixLength(*sort));
- } else if (!constraints[i]->GetConstraint<TEmptyConstraintNode>()) {
- commonPrefixLength = 0;
- }
- }
- if (commonPrefixLength) {
- return ctx.MakeConstraint<TSortedConstraintNode>(*sort, commonPrefixLength);
- }
-
- return nullptr;
-}
-
+ } else if (!constraints[i]->GetConstraint<TEmptyConstraintNode>()) {
+ commonPrefixLength = 0;
+ }
+ }
+ if (commonPrefixLength) {
+ return ctx.MakeConstraint<TSortedConstraintNode>(*sort, commonPrefixLength);
+ }
+
+ return nullptr;
+}
+
const TSortedConstraintNode* TSortedConstraintNode::FilterByType(const TSortedConstraintNode* sorted, const TStructExprType* outItemType, TExprContext& ctx) {
if (sorted) {
auto content = sorted->GetContent();
@@ -326,154 +326,154 @@ const TSortedConstraintNode* TSortedConstraintNode::CutPrefix(size_t newPrefixLe
//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-TGroupByConstraintNode::TGroupByConstraintNode(TExprContext& ctx, const TVector<TStringBuf>& columns)
- : TColumnSetConstraintNodeBase(ctx, Name(), columns)
-{
- YQL_ENSURE(!Columns_.empty());
-}
-
-TGroupByConstraintNode::TGroupByConstraintNode(TExprContext& ctx, const TVector<TString>& columns)
- : TColumnSetConstraintNodeBase(ctx, Name(), columns)
-{
- YQL_ENSURE(!Columns_.empty());
-}
-
-TGroupByConstraintNode::TGroupByConstraintNode(TExprContext& ctx, const TGroupByConstraintNode& constr, size_t prefixLength)
- : TColumnSetConstraintNodeBase(ctx, Name(), TVector<TStringBuf>(constr.GetColumns().begin(), constr.GetColumns().begin() + Min<size_t>(prefixLength, constr.GetColumns().size())))
-{
- YQL_ENSURE(!Columns_.empty());
- YQL_ENSURE(Columns_.size() == prefixLength);
-}
-
-TGroupByConstraintNode::TGroupByConstraintNode(TGroupByConstraintNode&& constr)
- : TColumnSetConstraintNodeBase(std::move(static_cast<TColumnSetConstraintNodeBase&>(constr)))
-{
-}
-
-size_t TGroupByConstraintNode::GetCommonPrefixLength(const TGroupByConstraintNode& node) const {
- const size_t minSize = Min(Columns_.size(), node.Columns_.size());
- for (size_t i = 0; i < minSize; ++i) {
- if (*(Columns_.begin() + i) != *(node.Columns_.begin() +i)) {
- return i;
- }
- }
- return minSize;
-}
-
-const TGroupByConstraintNode* TGroupByConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
- if (constraints.empty()) {
- return nullptr;
- }
-
- auto groupBy = constraints.front()->GetConstraint<TGroupByConstraintNode>();
- if (constraints.size() == 1 || !groupBy) {
- return groupBy;
- }
-
- size_t commonPrefixLength = groupBy->GetColumns().size();
- for (size_t i = 1; i < constraints.size() && commonPrefixLength > 0; ++i) {
- if (auto nextGroupBy = constraints[i]->GetConstraint<TGroupByConstraintNode>()) {
- commonPrefixLength = Min(commonPrefixLength, nextGroupBy->GetCommonPrefixLength(*groupBy));
- } else {
- commonPrefixLength = 0;
- }
- }
- if (commonPrefixLength) {
- return ctx.MakeConstraint<TGroupByConstraintNode>(*groupBy, commonPrefixLength);
- }
-
- return nullptr;
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-TUniqueConstraintNode::TUniqueConstraintNode(TExprContext& ctx, const TSetType& columns)
- : TColumnSetConstraintNodeBase(ctx, Name(), columns)
-{
- YQL_ENSURE(!Columns_.empty());
-}
-
-TUniqueConstraintNode::TUniqueConstraintNode(TExprContext& ctx, const TVector<TStringBuf>& columns)
- : TColumnSetConstraintNodeBase(ctx, Name(), columns)
-{
- YQL_ENSURE(!Columns_.empty());
-}
-
-TUniqueConstraintNode::TUniqueConstraintNode(TExprContext& ctx, const TVector<TString>& columns)
- : TColumnSetConstraintNodeBase(ctx, Name(), columns)
-{
- YQL_ENSURE(!Columns_.empty());
-}
-
-namespace {
-
-TUniqueConstraintNode::TSetType FromStructType(const TStructExprType& itemType) {
- TUniqueConstraintNode::TSetType res;
- for (auto& item: itemType.GetItems()) {
- res.push_back(item->GetName()); // Struct items are sorted
- }
- return res;
-}
-
-} // unnamed
-
-TUniqueConstraintNode::TUniqueConstraintNode(TExprContext& ctx, const TStructExprType& itemType)
- : TColumnSetConstraintNodeBase(ctx, Name(), FromStructType(itemType))
-{
- YQL_ENSURE(!Columns_.empty());
-}
-
-TUniqueConstraintNode::TUniqueConstraintNode(TUniqueConstraintNode&& constr)
- : TColumnSetConstraintNodeBase(std::move(static_cast<TColumnSetConstraintNodeBase&>(constr)))
-{
-}
-
-bool TUniqueConstraintNode::HasEqualColumns(TVector<TString> columns) const {
- ::SortUnique(columns);
- return columns.size() == Columns_.size() && std::equal(Columns_.begin(), Columns_.end(), columns.cbegin());
-}
-
-bool TUniqueConstraintNode::HasEqualColumns(TVector<TStringBuf> columns) const {
- ::SortUnique(columns);
- return columns.size() == Columns_.size() && std::equal(Columns_.begin(), Columns_.end(), columns.cbegin());
-}
-
-const TUniqueConstraintNode* TUniqueConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& /*ctx*/) {
- if (constraints.empty()) {
- return nullptr;
- }
- if (constraints.size() == 1) {
- return constraints.front()->GetConstraint<TUniqueConstraintNode>();
- }
-
- TVector<const TUniqueConstraintNode*> sorted;
- for (auto c: constraints) {
- if (auto uniq = c->GetConstraint<TUniqueConstraintNode>()) {
- sorted.push_back(uniq);
- } else if (!c->GetConstraint<TEmptyConstraintNode>()) {
- return nullptr;
- }
- }
- if (sorted.empty()) {
- return nullptr;
- }
- Sort(sorted, [](const TUniqueConstraintNode* c1, const TUniqueConstraintNode* c2) {
- return c1->GetColumns().size() > c2->GetColumns().size();
- });
- for (size_t i = 0; i + 1 < sorted.size(); ++i) {
- if (!sorted[i]->Includes(*sorted[i + 1])) {
- return nullptr;
- }
- }
-
- return sorted.front();
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
+TGroupByConstraintNode::TGroupByConstraintNode(TExprContext& ctx, const TVector<TStringBuf>& columns)
+ : TColumnSetConstraintNodeBase(ctx, Name(), columns)
+{
+ YQL_ENSURE(!Columns_.empty());
+}
+
+TGroupByConstraintNode::TGroupByConstraintNode(TExprContext& ctx, const TVector<TString>& columns)
+ : TColumnSetConstraintNodeBase(ctx, Name(), columns)
+{
+ YQL_ENSURE(!Columns_.empty());
+}
+
+TGroupByConstraintNode::TGroupByConstraintNode(TExprContext& ctx, const TGroupByConstraintNode& constr, size_t prefixLength)
+ : TColumnSetConstraintNodeBase(ctx, Name(), TVector<TStringBuf>(constr.GetColumns().begin(), constr.GetColumns().begin() + Min<size_t>(prefixLength, constr.GetColumns().size())))
+{
+ YQL_ENSURE(!Columns_.empty());
+ YQL_ENSURE(Columns_.size() == prefixLength);
+}
+
+TGroupByConstraintNode::TGroupByConstraintNode(TGroupByConstraintNode&& constr)
+ : TColumnSetConstraintNodeBase(std::move(static_cast<TColumnSetConstraintNodeBase&>(constr)))
+{
+}
+
+size_t TGroupByConstraintNode::GetCommonPrefixLength(const TGroupByConstraintNode& node) const {
+ const size_t minSize = Min(Columns_.size(), node.Columns_.size());
+ for (size_t i = 0; i < minSize; ++i) {
+ if (*(Columns_.begin() + i) != *(node.Columns_.begin() +i)) {
+ return i;
+ }
+ }
+ return minSize;
+}
+
+const TGroupByConstraintNode* TGroupByConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
+ if (constraints.empty()) {
+ return nullptr;
+ }
+
+ auto groupBy = constraints.front()->GetConstraint<TGroupByConstraintNode>();
+ if (constraints.size() == 1 || !groupBy) {
+ return groupBy;
+ }
+
+ size_t commonPrefixLength = groupBy->GetColumns().size();
+ for (size_t i = 1; i < constraints.size() && commonPrefixLength > 0; ++i) {
+ if (auto nextGroupBy = constraints[i]->GetConstraint<TGroupByConstraintNode>()) {
+ commonPrefixLength = Min(commonPrefixLength, nextGroupBy->GetCommonPrefixLength(*groupBy));
+ } else {
+ commonPrefixLength = 0;
+ }
+ }
+ if (commonPrefixLength) {
+ return ctx.MakeConstraint<TGroupByConstraintNode>(*groupBy, commonPrefixLength);
+ }
+
+ return nullptr;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TUniqueConstraintNode::TUniqueConstraintNode(TExprContext& ctx, const TSetType& columns)
+ : TColumnSetConstraintNodeBase(ctx, Name(), columns)
+{
+ YQL_ENSURE(!Columns_.empty());
+}
+
+TUniqueConstraintNode::TUniqueConstraintNode(TExprContext& ctx, const TVector<TStringBuf>& columns)
+ : TColumnSetConstraintNodeBase(ctx, Name(), columns)
+{
+ YQL_ENSURE(!Columns_.empty());
+}
+
+TUniqueConstraintNode::TUniqueConstraintNode(TExprContext& ctx, const TVector<TString>& columns)
+ : TColumnSetConstraintNodeBase(ctx, Name(), columns)
+{
+ YQL_ENSURE(!Columns_.empty());
+}
+
+namespace {
+
+TUniqueConstraintNode::TSetType FromStructType(const TStructExprType& itemType) {
+ TUniqueConstraintNode::TSetType res;
+ for (auto& item: itemType.GetItems()) {
+ res.push_back(item->GetName()); // Struct items are sorted
+ }
+ return res;
+}
+
+} // unnamed
+
+TUniqueConstraintNode::TUniqueConstraintNode(TExprContext& ctx, const TStructExprType& itemType)
+ : TColumnSetConstraintNodeBase(ctx, Name(), FromStructType(itemType))
+{
+ YQL_ENSURE(!Columns_.empty());
+}
+
+TUniqueConstraintNode::TUniqueConstraintNode(TUniqueConstraintNode&& constr)
+ : TColumnSetConstraintNodeBase(std::move(static_cast<TColumnSetConstraintNodeBase&>(constr)))
+{
+}
+
+bool TUniqueConstraintNode::HasEqualColumns(TVector<TString> columns) const {
+ ::SortUnique(columns);
+ return columns.size() == Columns_.size() && std::equal(Columns_.begin(), Columns_.end(), columns.cbegin());
+}
+
+bool TUniqueConstraintNode::HasEqualColumns(TVector<TStringBuf> columns) const {
+ ::SortUnique(columns);
+ return columns.size() == Columns_.size() && std::equal(Columns_.begin(), Columns_.end(), columns.cbegin());
+}
+
+const TUniqueConstraintNode* TUniqueConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& /*ctx*/) {
+ if (constraints.empty()) {
+ return nullptr;
+ }
+ if (constraints.size() == 1) {
+ return constraints.front()->GetConstraint<TUniqueConstraintNode>();
+ }
+
+ TVector<const TUniqueConstraintNode*> sorted;
+ for (auto c: constraints) {
+ if (auto uniq = c->GetConstraint<TUniqueConstraintNode>()) {
+ sorted.push_back(uniq);
+ } else if (!c->GetConstraint<TEmptyConstraintNode>()) {
+ return nullptr;
+ }
+ }
+ if (sorted.empty()) {
+ return nullptr;
+ }
+ Sort(sorted, [](const TUniqueConstraintNode* c1, const TUniqueConstraintNode* c2) {
+ return c1->GetColumns().size() > c2->GetColumns().size();
+ });
+ for (size_t i = 0; i + 1 < sorted.size(); ++i) {
+ if (!sorted[i]->Includes(*sorted[i + 1])) {
+ return nullptr;
+ }
+ }
+
+ return sorted.front();
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
TPassthroughConstraintNode::TPassthroughConstraintNode(TExprContext& ctx, TMapType&& mapping)
: TConstraintNode(ctx, Name()), Mapping_(std::move(mapping))
-{
+{
YQL_ENSURE(!Mapping_.empty());
if (const auto original = Mapping_.cbegin()->first;
original && 1U == Mapping_.size() && 1U == original->Mapping_.size() && Mapping_.cbegin()->second == original->Mapping_.cbegin()->second) {
@@ -490,23 +490,23 @@ TPassthroughConstraintNode::TPassthroughConstraintNode(TExprContext& ctx, TMapTy
Hash_ = MurmurHash<ui64>(item.second.data(), item.second.size(), Hash_);
}
}
- }
-}
-
-TPassthroughConstraintNode::TPassthroughConstraintNode(TExprContext& ctx, const TStructExprType& itemType)
- : TConstraintNode(ctx, Name())
-{
+ }
+}
+
+TPassthroughConstraintNode::TPassthroughConstraintNode(TExprContext& ctx, const TStructExprType& itemType)
+ : TConstraintNode(ctx, Name())
+{
auto& mapping = Mapping_[nullptr];
- for (auto& item: itemType.GetItems()) {
+ for (auto& item: itemType.GetItems()) {
const auto name = ctx.AppendString(item->GetName());
- Hash_ = MurmurHash<ui64>(name.data(), name.size(), Hash_); // hash as name
- Hash_ = MurmurHash<ui64>(name.data(), name.size(), Hash_); // hash as value
+ Hash_ = MurmurHash<ui64>(name.data(), name.size(), Hash_); // hash as name
+ Hash_ = MurmurHash<ui64>(name.data(), name.size(), Hash_); // hash as value
mapping.push_back(std::make_pair(TKeyType(1U, name), name)); // Struct items are sorted
- }
-
+ }
+
YQL_ENSURE(!mapping.empty());
-}
-
+}
+
TPassthroughConstraintNode::TPassthroughConstraintNode(TExprContext& ctx, const TTupleExprType& itemType)
: TConstraintNode(ctx, Name())
{
@@ -536,24 +536,24 @@ TPassthroughConstraintNode::TPassthroughConstraintNode(TExprContext& ctx, const
}
TPassthroughConstraintNode::TPassthroughConstraintNode(TPassthroughConstraintNode&& constr) = default;
-
-bool TPassthroughConstraintNode::Equals(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
- if (GetHash() != node.GetHash()) {
- return false;
- }
+
+bool TPassthroughConstraintNode::Equals(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
+ if (GetHash() != node.GetHash()) {
+ return false;
+ }
if (const auto c = dynamic_cast<const TPassthroughConstraintNode*>(&node)) {
return Mapping_ == c->Mapping_;
- }
- return false;
-}
-
-bool TPassthroughConstraintNode::Includes(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
+ }
+ return false;
+}
+
+bool TPassthroughConstraintNode::Includes(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
if (const auto c = dynamic_cast<const TPassthroughConstraintNode*>(&node)) {
for (const auto& part : c->Mapping_) {
if (const auto it = Mapping_.find(part.first == this ? nullptr : part.first); Mapping_.cend() != it) {
@@ -561,20 +561,20 @@ bool TPassthroughConstraintNode::Includes(const TConstraintNode& node) const {
if (const auto p = it->second.find(pair.first); it->second.cend() == p || p->second != pair.second) {
return false;
}
- }
+ }
} else
- return false;
- }
- return true;
- }
- return false;
-}
-
-void TPassthroughConstraintNode::Out(IOutputStream& out) const {
- TConstraintNode::Out(out);
- out.Write('(');
-
- bool first = true;
+ return false;
+ }
+ return true;
+ }
+ return false;
+}
+
+void TPassthroughConstraintNode::Out(IOutputStream& out) const {
+ TConstraintNode::Out(out);
+ out.Write('(');
+
+ bool first = true;
for (const auto& part : Mapping_) {
for (const auto& item : part.second) {
if (!first) {
@@ -592,11 +592,11 @@ void TPassthroughConstraintNode::Out(IOutputStream& out) const {
out.Write(item.second);
first = false;
- }
- }
- out.Write(')');
-}
-
+ }
+ }
+ out.Write(')');
+}
+
void TPassthroughConstraintNode::ToJson(NJson::TJsonWriter& out) const {
out.OpenMap();
for (const auto& part : Mapping_) {
@@ -633,28 +633,28 @@ const TPassthroughConstraintNode* TPassthroughConstraintNode::ExtractField(TExpr
return passtrought.empty() ? nullptr : ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(passtrought));
}
-const TPassthroughConstraintNode* TPassthroughConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
- if (constraints.empty()) {
- return nullptr;
- }
-
+const TPassthroughConstraintNode* TPassthroughConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
+ if (constraints.empty()) {
+ return nullptr;
+ }
+
auto part = constraints.front()->GetConstraint<TPassthroughConstraintNode>();
if (constraints.size() == 1 || !part) {
return part;
- }
-
+ }
+
auto mapping = part->GetColumnMapping();
if (const auto self = mapping.find(nullptr); mapping.cend() != self)
mapping.emplace(part, std::move(mapping.extract(self).mapped()));
- for (size_t i = 1; i < constraints.size() && !mapping.empty(); ++i) {
+ for (size_t i = 1; i < constraints.size() && !mapping.empty(); ++i) {
part = constraints[i]->GetConstraint<TPassthroughConstraintNode>();
if (!part) {
- if (constraints[i]->GetConstraint<TEmptyConstraintNode>()) {
- continue;
- }
- return nullptr;
- }
+ if (constraints[i]->GetConstraint<TEmptyConstraintNode>()) {
+ continue;
+ }
+ return nullptr;
+ }
for (const auto& nextMapping : part->GetColumnMapping()) {
if (const auto it = mapping.find(nextMapping.first ? nextMapping.first : part); mapping.cend() != it) {
@@ -671,9 +671,9 @@ const TPassthroughConstraintNode* TPassthroughConstraintNode::MakeCommon(const T
mapping.erase(it);
else
it->second = std::move(result);
- }
+ }
}
- }
+ }
return mapping.empty() ? nullptr : ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(mapping));
}
@@ -693,144 +693,144 @@ TPassthroughConstraintNode::TReverseMapType TPassthroughConstraintNode::GetRever
reverseMapping.emplace_back(item.second, item.first.front());
}
}
- }
+ }
::Sort(reverseMapping);
return reverseMapping;
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-TEmptyConstraintNode::TEmptyConstraintNode(TExprContext& ctx)
- : TConstraintNode(ctx, Name())
-{
-}
-
-TEmptyConstraintNode::TEmptyConstraintNode(TEmptyConstraintNode&& constr)
- : TConstraintNode(std::move(static_cast<TConstraintNode&>(constr)))
-{
-}
-
-bool TEmptyConstraintNode::Equals(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
- if (GetHash() != node.GetHash()) {
- return false;
- }
- return GetName() == node.GetName();
-}
-
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TEmptyConstraintNode::TEmptyConstraintNode(TExprContext& ctx)
+ : TConstraintNode(ctx, Name())
+{
+}
+
+TEmptyConstraintNode::TEmptyConstraintNode(TEmptyConstraintNode&& constr)
+ : TConstraintNode(std::move(static_cast<TConstraintNode&>(constr)))
+{
+}
+
+bool TEmptyConstraintNode::Equals(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
+ if (GetHash() != node.GetHash()) {
+ return false;
+ }
+ return GetName() == node.GetName();
+}
+
void TEmptyConstraintNode::ToJson(NJson::TJsonWriter& out) const {
out.Write(true);
}
-const TEmptyConstraintNode* TEmptyConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& /*ctx*/) {
- if (constraints.empty()) {
- return nullptr;
- }
-
- auto empty = constraints.front()->GetConstraint<TEmptyConstraintNode>();
- if (AllOf(constraints.cbegin() + 1, constraints.cend(), [empty](const TConstraintSet* c) { return c->GetConstraint<TEmptyConstraintNode>() == empty; })) {
- return empty;
- }
- return nullptr;
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-TVarIndexConstraintNode::TVarIndexConstraintNode(TExprContext& ctx, const TMapType& mapping)
- : TConstraintNode(ctx, Name())
- , Mapping_(mapping)
-{
- Hash_ = MurmurHash<ui64>(Mapping_.data(), Mapping_.size() * sizeof(TMapType::value_type), Hash_);
- YQL_ENSURE(!Mapping_.empty());
-}
-
-TVarIndexConstraintNode::TVarIndexConstraintNode(TExprContext& ctx, const TVariantExprType& itemType)
- : TVarIndexConstraintNode(ctx, itemType.GetUnderlyingType()->Cast<TTupleExprType>()->GetSize())
-{
-}
-
-TVarIndexConstraintNode::TVarIndexConstraintNode(TExprContext& ctx, size_t mapItemsCount)
- : TConstraintNode(ctx, Name())
-{
- YQL_ENSURE(mapItemsCount > 0);
- for (size_t i = 0; i < mapItemsCount; ++i) {
- Mapping_.push_back(std::make_pair(i, i));
- }
- Hash_ = MurmurHash<ui64>(Mapping_.data(), Mapping_.size() * sizeof(TMapType::value_type), Hash_);
- YQL_ENSURE(!Mapping_.empty());
-}
-
-TVarIndexConstraintNode::TVarIndexConstraintNode(TVarIndexConstraintNode&& constr)
- : TConstraintNode(std::move(static_cast<TConstraintNode&>(constr)))
- , Mapping_(std::move(constr.Mapping_))
-{
-}
-
-TVarIndexConstraintNode::TMapType TVarIndexConstraintNode::GetReverseMapping() const {
- TMapType reverseMapping;
- std::transform(Mapping_.cbegin(), Mapping_.cend(),
- std::back_inserter(reverseMapping),
- [] (const std::pair<size_t, size_t>& p) { return std::make_pair(p.second, p.first); }
- );
- ::Sort(reverseMapping);
- return reverseMapping;
-}
-
-bool TVarIndexConstraintNode::Equals(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
- if (GetHash() != node.GetHash()) {
- return false;
- }
- if (GetName() != node.GetName()) {
- return false;
- }
- if (auto c = dynamic_cast<const TVarIndexConstraintNode*>(&node)) {
- return GetIndexMapping() == c->GetIndexMapping();
- }
- return false;
-}
-
-bool TVarIndexConstraintNode::Includes(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
- if (GetName() != node.GetName()) {
- return false;
- }
- if (auto c = dynamic_cast<const TVarIndexConstraintNode*>(&node)) {
- for (auto& pair: c->Mapping_) {
- if (auto p = Mapping_.FindPtr(pair.first)) {
- if (*p != pair.second) {
- return false;
- }
- } else {
- return false;
- }
- }
- return true;
- }
- return false;
-}
-
-void TVarIndexConstraintNode::Out(IOutputStream& out) const {
- TConstraintNode::Out(out);
- out.Write('(');
-
- bool first = true;
- for (auto& item: Mapping_) {
- if (!first) {
- out.Write(',');
- }
- out << item.first << ':' << item.second;
- first = false;
- }
- out.Write(')');
-}
-
+const TEmptyConstraintNode* TEmptyConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& /*ctx*/) {
+ if (constraints.empty()) {
+ return nullptr;
+ }
+
+ auto empty = constraints.front()->GetConstraint<TEmptyConstraintNode>();
+ if (AllOf(constraints.cbegin() + 1, constraints.cend(), [empty](const TConstraintSet* c) { return c->GetConstraint<TEmptyConstraintNode>() == empty; })) {
+ return empty;
+ }
+ return nullptr;
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TVarIndexConstraintNode::TVarIndexConstraintNode(TExprContext& ctx, const TMapType& mapping)
+ : TConstraintNode(ctx, Name())
+ , Mapping_(mapping)
+{
+ Hash_ = MurmurHash<ui64>(Mapping_.data(), Mapping_.size() * sizeof(TMapType::value_type), Hash_);
+ YQL_ENSURE(!Mapping_.empty());
+}
+
+TVarIndexConstraintNode::TVarIndexConstraintNode(TExprContext& ctx, const TVariantExprType& itemType)
+ : TVarIndexConstraintNode(ctx, itemType.GetUnderlyingType()->Cast<TTupleExprType>()->GetSize())
+{
+}
+
+TVarIndexConstraintNode::TVarIndexConstraintNode(TExprContext& ctx, size_t mapItemsCount)
+ : TConstraintNode(ctx, Name())
+{
+ YQL_ENSURE(mapItemsCount > 0);
+ for (size_t i = 0; i < mapItemsCount; ++i) {
+ Mapping_.push_back(std::make_pair(i, i));
+ }
+ Hash_ = MurmurHash<ui64>(Mapping_.data(), Mapping_.size() * sizeof(TMapType::value_type), Hash_);
+ YQL_ENSURE(!Mapping_.empty());
+}
+
+TVarIndexConstraintNode::TVarIndexConstraintNode(TVarIndexConstraintNode&& constr)
+ : TConstraintNode(std::move(static_cast<TConstraintNode&>(constr)))
+ , Mapping_(std::move(constr.Mapping_))
+{
+}
+
+TVarIndexConstraintNode::TMapType TVarIndexConstraintNode::GetReverseMapping() const {
+ TMapType reverseMapping;
+ std::transform(Mapping_.cbegin(), Mapping_.cend(),
+ std::back_inserter(reverseMapping),
+ [] (const std::pair<size_t, size_t>& p) { return std::make_pair(p.second, p.first); }
+ );
+ ::Sort(reverseMapping);
+ return reverseMapping;
+}
+
+bool TVarIndexConstraintNode::Equals(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
+ if (GetHash() != node.GetHash()) {
+ return false;
+ }
+ if (GetName() != node.GetName()) {
+ return false;
+ }
+ if (auto c = dynamic_cast<const TVarIndexConstraintNode*>(&node)) {
+ return GetIndexMapping() == c->GetIndexMapping();
+ }
+ return false;
+}
+
+bool TVarIndexConstraintNode::Includes(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
+ if (GetName() != node.GetName()) {
+ return false;
+ }
+ if (auto c = dynamic_cast<const TVarIndexConstraintNode*>(&node)) {
+ for (auto& pair: c->Mapping_) {
+ if (auto p = Mapping_.FindPtr(pair.first)) {
+ if (*p != pair.second) {
+ return false;
+ }
+ } else {
+ return false;
+ }
+ }
+ return true;
+ }
+ return false;
+}
+
+void TVarIndexConstraintNode::Out(IOutputStream& out) const {
+ TConstraintNode::Out(out);
+ out.Write('(');
+
+ bool first = true;
+ for (auto& item: Mapping_) {
+ if (!first) {
+ out.Write(',');
+ }
+ out << item.first << ':' << item.second;
+ first = false;
+ }
+ out.Write(')');
+}
+
void TVarIndexConstraintNode::ToJson(NJson::TJsonWriter& out) const {
out.OpenArray();
for (const auto& [resultIndex, originalIndex]: Mapping_) {
@@ -842,104 +842,104 @@ void TVarIndexConstraintNode::ToJson(NJson::TJsonWriter& out) const {
out.CloseArray();
}
-const TVarIndexConstraintNode* TVarIndexConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
- if (constraints.empty()) {
- return nullptr;
- }
-
- if (constraints.size() == 1) {
- return constraints.front()->GetConstraint<TVarIndexConstraintNode>();
- }
-
- TVarIndexConstraintNode::TMapType mapping;
- for (size_t i = 0; i < constraints.size(); ++i) {
- if (auto varIndex = constraints[i]->GetConstraint<TVarIndexConstraintNode>()) {
+const TVarIndexConstraintNode* TVarIndexConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
+ if (constraints.empty()) {
+ return nullptr;
+ }
+
+ if (constraints.size() == 1) {
+ return constraints.front()->GetConstraint<TVarIndexConstraintNode>();
+ }
+
+ TVarIndexConstraintNode::TMapType mapping;
+ for (size_t i = 0; i < constraints.size(); ++i) {
+ if (auto varIndex = constraints[i]->GetConstraint<TVarIndexConstraintNode>()) {
mapping.insert(varIndex->GetIndexMapping().begin(), varIndex->GetIndexMapping().end());
- }
- }
- if (mapping.empty()) {
- return nullptr;
- }
- ::SortUnique(mapping);
- return ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(mapping));
-}
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-TMultiConstraintNode::TMultiConstraintNode(TExprContext& ctx, const TMapType& items)
- : TConstraintNode(ctx, Name())
-{
- YQL_ENSURE(items.size());
- for (auto& item: items) {
- Hash_ = MurmurHash<ui64>(&item.first, sizeof(item.first), Hash_);
- for (auto c: item.second.GetAllConstraints()) {
- const auto itemHash = c->GetHash();
- Hash_ = MurmurHash<ui64>(&itemHash, sizeof(itemHash), Hash_);
- }
- }
- Items_ = items;
-}
-
-TMultiConstraintNode::TMultiConstraintNode(TExprContext& ctx, ui32 index, const TConstraintSet& constraints)
- : TMultiConstraintNode(ctx, TMapType{{index, constraints}})
-{
-}
-
-TMultiConstraintNode::TMultiConstraintNode(TMultiConstraintNode&& constr)
- : TConstraintNode(std::move(static_cast<TConstraintNode&>(constr)))
- , Items_(std::move(constr.Items_))
-{
-}
-
-bool TMultiConstraintNode::Equals(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
- if (GetHash() != node.GetHash()) {
- return false;
- }
- if (GetName() != node.GetName()) {
- return false;
- }
- if (auto c = dynamic_cast<const TMultiConstraintNode*>(&node)) {
- return GetItems() == c->GetItems();
- }
- return false;
-}
-
-bool TMultiConstraintNode::Includes(const TConstraintNode& node) const {
- if (this == &node) {
- return true;
- }
- if (GetName() != node.GetName()) {
- return false;
- }
-
- if (auto m = dynamic_cast<const TMultiConstraintNode*>(&node)) {
- for (auto& item: Items_) {
- const auto it = m->Items_.find(item.first);
- if (it == m->Items_.end()) {
- if (!item.second.GetConstraint<TEmptyConstraintNode>()) {
- return false;
- }
- continue;
- }
-
- for (auto c: it->second.GetAllConstraints()) {
- auto cit = item.second.GetConstraint(c->GetName());
- if (!cit) {
- return false;
- }
- if (!cit->Includes(*c)) {
- return false;
- }
- }
- }
- return true;
- }
- return false;
-}
-
+ }
+ }
+ if (mapping.empty()) {
+ return nullptr;
+ }
+ ::SortUnique(mapping);
+ return ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(mapping));
+}
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+TMultiConstraintNode::TMultiConstraintNode(TExprContext& ctx, const TMapType& items)
+ : TConstraintNode(ctx, Name())
+{
+ YQL_ENSURE(items.size());
+ for (auto& item: items) {
+ Hash_ = MurmurHash<ui64>(&item.first, sizeof(item.first), Hash_);
+ for (auto c: item.second.GetAllConstraints()) {
+ const auto itemHash = c->GetHash();
+ Hash_ = MurmurHash<ui64>(&itemHash, sizeof(itemHash), Hash_);
+ }
+ }
+ Items_ = items;
+}
+
+TMultiConstraintNode::TMultiConstraintNode(TExprContext& ctx, ui32 index, const TConstraintSet& constraints)
+ : TMultiConstraintNode(ctx, TMapType{{index, constraints}})
+{
+}
+
+TMultiConstraintNode::TMultiConstraintNode(TMultiConstraintNode&& constr)
+ : TConstraintNode(std::move(static_cast<TConstraintNode&>(constr)))
+ , Items_(std::move(constr.Items_))
+{
+}
+
+bool TMultiConstraintNode::Equals(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
+ if (GetHash() != node.GetHash()) {
+ return false;
+ }
+ if (GetName() != node.GetName()) {
+ return false;
+ }
+ if (auto c = dynamic_cast<const TMultiConstraintNode*>(&node)) {
+ return GetItems() == c->GetItems();
+ }
+ return false;
+}
+
+bool TMultiConstraintNode::Includes(const TConstraintNode& node) const {
+ if (this == &node) {
+ return true;
+ }
+ if (GetName() != node.GetName()) {
+ return false;
+ }
+
+ if (auto m = dynamic_cast<const TMultiConstraintNode*>(&node)) {
+ for (auto& item: Items_) {
+ const auto it = m->Items_.find(item.first);
+ if (it == m->Items_.end()) {
+ if (!item.second.GetConstraint<TEmptyConstraintNode>()) {
+ return false;
+ }
+ continue;
+ }
+
+ for (auto c: it->second.GetAllConstraints()) {
+ auto cit = item.second.GetConstraint(c->GetName());
+ if (!cit) {
+ return false;
+ }
+ if (!cit->Includes(*c)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+ return false;
+}
+
bool TMultiConstraintNode::FilteredIncludes(const TConstraintNode& node, const THashSet<TString>& blacklist) const {
if (this == &node) {
return true;
@@ -975,29 +975,29 @@ bool TMultiConstraintNode::FilteredIncludes(const TConstraintNode& node, const T
return false;
}
-void TMultiConstraintNode::Out(IOutputStream& out) const {
- TConstraintNode::Out(out);
- out.Write('(');
- bool first = true;
- for (auto& item: Items_) {
- if (!first) {
- out.Write(',');
- }
- out << item.first << ':' << '{';
- bool firstConstr = true;
- for (auto c: item.second.GetAllConstraints()) {
- if (!firstConstr) {
- out.Write(',');
- }
- out << *c;
- firstConstr = false;
- }
- out << '}';
- first = false;
- }
- out.Write(')');
-}
-
+void TMultiConstraintNode::Out(IOutputStream& out) const {
+ TConstraintNode::Out(out);
+ out.Write('(');
+ bool first = true;
+ for (auto& item: Items_) {
+ if (!first) {
+ out.Write(',');
+ }
+ out << item.first << ':' << '{';
+ bool firstConstr = true;
+ for (auto c: item.second.GetAllConstraints()) {
+ if (!firstConstr) {
+ out.Write(',');
+ }
+ out << *c;
+ firstConstr = false;
+ }
+ out << '}';
+ first = false;
+ }
+ out.Write(')');
+}
+
void TMultiConstraintNode::ToJson(NJson::TJsonWriter& out) const {
out.OpenMap();
for (const auto& [index, constraintSet] : Items_) {
@@ -1007,108 +1007,108 @@ void TMultiConstraintNode::ToJson(NJson::TJsonWriter& out) const {
out.CloseMap();
}
-const TMultiConstraintNode* TMultiConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
- if (constraints.empty()) {
- return nullptr;
- } else if (constraints.size() == 1) {
- return constraints.front()->GetConstraint<TMultiConstraintNode>();
- }
-
- TMultiConstraintNode::TMapType multiItems;
- for (auto c: constraints) {
- if (auto m = c->GetConstraint<TMultiConstraintNode>()) {
+const TMultiConstraintNode* TMultiConstraintNode::MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
+ if (constraints.empty()) {
+ return nullptr;
+ } else if (constraints.size() == 1) {
+ return constraints.front()->GetConstraint<TMultiConstraintNode>();
+ }
+
+ TMultiConstraintNode::TMapType multiItems;
+ for (auto c: constraints) {
+ if (auto m = c->GetConstraint<TMultiConstraintNode>()) {
multiItems.insert(m->GetItems().begin(), m->GetItems().end());
- } else if (!c->GetConstraint<TEmptyConstraintNode>()) {
- return nullptr;
- }
- }
- if (multiItems.empty()) {
- return nullptr;
- }
-
- multiItems.sort();
- // Remove duplicates
- // For duplicated items keep only Empty constraint
- auto cur = multiItems.begin();
- while (cur != multiItems.end()) {
- auto start = cur;
- do {
- ++cur;
- } while (cur != multiItems.end() && start->first == cur->first);
-
- switch (std::distance(start, cur)) {
- case 0:
- break;
- case 1:
- if (start->second.GetConstraint<TEmptyConstraintNode>()) {
- cur = multiItems.erase(start, cur);
- }
- break;
- default:
- {
- std::vector<TMultiConstraintNode::TMapType::value_type> nonEmpty;
- std::copy_if(start, cur, std::back_inserter(nonEmpty),
- [] (const TMultiConstraintNode::TMapType::value_type& v) {
- return !v.second.GetConstraint<TEmptyConstraintNode>();
- }
- );
- start->second.Clear();
- if (nonEmpty.empty()) {
- start->second.AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- } else if (nonEmpty.size() == 1) {
- start->second = nonEmpty.front().second;
- }
- cur = multiItems.erase(start + 1, cur);
- }
- }
- }
- if (!multiItems.empty()) {
- return ctx.MakeConstraint<TMultiConstraintNode>(std::move(multiItems));
- }
-
- return nullptr;
-}
-
-} // namespace NYql
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-template<>
-void Out<NYql::TConstraintNode>(IOutputStream& out, const NYql::TConstraintNode& c) {
- c.Out(out);
-}
-
-template<>
-void Out<NYql::TSortedConstraintNode>(IOutputStream& out, const NYql::TSortedConstraintNode& c) {
- c.Out(out);
-}
-
-template<>
-void Out<NYql::TGroupByConstraintNode>(IOutputStream& out, const NYql::TGroupByConstraintNode& c) {
- c.Out(out);
-}
-
-template<>
-void Out<NYql::TUniqueConstraintNode>(IOutputStream& out, const NYql::TUniqueConstraintNode& c) {
- c.Out(out);
-}
-
-template<>
-void Out<NYql::TPassthroughConstraintNode>(IOutputStream& out, const NYql::TPassthroughConstraintNode& c) {
- c.Out(out);
-}
-
-template<>
-void Out<NYql::TEmptyConstraintNode>(IOutputStream& out, const NYql::TEmptyConstraintNode& c) {
- c.Out(out);
-}
-
-template<>
-void Out<NYql::TVarIndexConstraintNode>(IOutputStream& out, const NYql::TVarIndexConstraintNode& c) {
- c.Out(out);
-}
-
-template<>
-void Out<NYql::TMultiConstraintNode>(IOutputStream& out, const NYql::TMultiConstraintNode& c) {
- c.Out(out);
-}
+ } else if (!c->GetConstraint<TEmptyConstraintNode>()) {
+ return nullptr;
+ }
+ }
+ if (multiItems.empty()) {
+ return nullptr;
+ }
+
+ multiItems.sort();
+ // Remove duplicates
+ // For duplicated items keep only Empty constraint
+ auto cur = multiItems.begin();
+ while (cur != multiItems.end()) {
+ auto start = cur;
+ do {
+ ++cur;
+ } while (cur != multiItems.end() && start->first == cur->first);
+
+ switch (std::distance(start, cur)) {
+ case 0:
+ break;
+ case 1:
+ if (start->second.GetConstraint<TEmptyConstraintNode>()) {
+ cur = multiItems.erase(start, cur);
+ }
+ break;
+ default:
+ {
+ std::vector<TMultiConstraintNode::TMapType::value_type> nonEmpty;
+ std::copy_if(start, cur, std::back_inserter(nonEmpty),
+ [] (const TMultiConstraintNode::TMapType::value_type& v) {
+ return !v.second.GetConstraint<TEmptyConstraintNode>();
+ }
+ );
+ start->second.Clear();
+ if (nonEmpty.empty()) {
+ start->second.AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ } else if (nonEmpty.size() == 1) {
+ start->second = nonEmpty.front().second;
+ }
+ cur = multiItems.erase(start + 1, cur);
+ }
+ }
+ }
+ if (!multiItems.empty()) {
+ return ctx.MakeConstraint<TMultiConstraintNode>(std::move(multiItems));
+ }
+
+ return nullptr;
+}
+
+} // namespace NYql
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+template<>
+void Out<NYql::TConstraintNode>(IOutputStream& out, const NYql::TConstraintNode& c) {
+ c.Out(out);
+}
+
+template<>
+void Out<NYql::TSortedConstraintNode>(IOutputStream& out, const NYql::TSortedConstraintNode& c) {
+ c.Out(out);
+}
+
+template<>
+void Out<NYql::TGroupByConstraintNode>(IOutputStream& out, const NYql::TGroupByConstraintNode& c) {
+ c.Out(out);
+}
+
+template<>
+void Out<NYql::TUniqueConstraintNode>(IOutputStream& out, const NYql::TUniqueConstraintNode& c) {
+ c.Out(out);
+}
+
+template<>
+void Out<NYql::TPassthroughConstraintNode>(IOutputStream& out, const NYql::TPassthroughConstraintNode& c) {
+ c.Out(out);
+}
+
+template<>
+void Out<NYql::TEmptyConstraintNode>(IOutputStream& out, const NYql::TEmptyConstraintNode& c) {
+ c.Out(out);
+}
+
+template<>
+void Out<NYql::TVarIndexConstraintNode>(IOutputStream& out, const NYql::TVarIndexConstraintNode& c) {
+ c.Out(out);
+}
+
+template<>
+void Out<NYql::TMultiConstraintNode>(IOutputStream& out, const NYql::TMultiConstraintNode& c) {
+ c.Out(out);
+}
diff --git a/ydb/library/yql/ast/yql_constraint.h b/ydb/library/yql/ast/yql_constraint.h
index c093f3c9f4..5d0440bbfc 100644
--- a/ydb/library/yql/ast/yql_constraint.h
+++ b/ydb/library/yql/ast/yql_constraint.h
@@ -1,185 +1,185 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/utils/yql_panic.h>
-
+
#include <library/cpp/containers/stack_vector/stack_vec.h>
-#include <library/cpp/containers/sorted_vector/sorted_vector.h>
+#include <library/cpp/containers/sorted_vector/sorted_vector.h>
#include <library/cpp/json/json_writer.h>
-
-#include <util/generic/strbuf.h>
-#include <util/generic/string.h>
-#include <util/stream/output.h>
-
+
+#include <util/generic/strbuf.h>
+#include <util/generic/string.h>
+#include <util/stream/output.h>
+
#include <deque>
#include <unordered_map>
-
-namespace NYql {
-
-struct TExprContext;
-class TStructExprType;
+
+namespace NYql {
+
+struct TExprContext;
+class TStructExprType;
class TTupleExprType;
class TMultiExprType;
-class TVariantExprType;
-
-class TConstraintNode {
-protected:
+class TVariantExprType;
+
+class TConstraintNode {
+protected:
TConstraintNode(TExprContext& ctx, std::string_view name);
- TConstraintNode(TConstraintNode&& constr);
-
-public:
- using TListType = std::vector<const TConstraintNode*>;
-
- struct THash {
- size_t operator()(const TConstraintNode* node) const {
- return node->GetHash();
- }
- };
-
- struct TEqual {
- bool operator()(const TConstraintNode* one, const TConstraintNode* two) const {
- return one->Equals(*two);
- }
- };
-
- struct TCompare {
- inline bool operator()(const TConstraintNode* l, const TConstraintNode* r) const {
- return l->GetName() < r->GetName();
- }
-
+ TConstraintNode(TConstraintNode&& constr);
+
+public:
+ using TListType = std::vector<const TConstraintNode*>;
+
+ struct THash {
+ size_t operator()(const TConstraintNode* node) const {
+ return node->GetHash();
+ }
+ };
+
+ struct TEqual {
+ bool operator()(const TConstraintNode* one, const TConstraintNode* two) const {
+ return one->Equals(*two);
+ }
+ };
+
+ struct TCompare {
+ inline bool operator()(const TConstraintNode* l, const TConstraintNode* r) const {
+ return l->GetName() < r->GetName();
+ }
+
inline bool operator()(const std::string_view name, const TConstraintNode* r) const {
- return name < r->GetName();
- }
-
+ return name < r->GetName();
+ }
+
inline bool operator()(const TConstraintNode* l, const std::string_view name) const {
- return l->GetName() < name;
- }
- };
-
- virtual ~TConstraintNode() = default;
-
- ui64 GetHash() const {
- return Hash_;
- }
-
- virtual bool Equals(const TConstraintNode& node) const = 0;
- virtual bool Includes(const TConstraintNode& node) const {
- return Equals(node);
- }
- virtual void Out(IOutputStream& out) const;
+ return l->GetName() < name;
+ }
+ };
+
+ virtual ~TConstraintNode() = default;
+
+ ui64 GetHash() const {
+ return Hash_;
+ }
+
+ virtual bool Equals(const TConstraintNode& node) const = 0;
+ virtual bool Includes(const TConstraintNode& node) const {
+ return Equals(node);
+ }
+ virtual void Out(IOutputStream& out) const;
virtual void ToJson(NJson::TJsonWriter& out) const = 0;
-
- template <typename T>
- const T* Cast() const {
- static_assert(std::is_base_of<TConstraintNode, T>::value,
- "Should be derived from TConstraintNode");
-
- const auto ret = dynamic_cast<const T*>(this);
+
+ template <typename T>
+ const T* Cast() const {
+ static_assert(std::is_base_of<TConstraintNode, T>::value,
+ "Should be derived from TConstraintNode");
+
+ const auto ret = dynamic_cast<const T*>(this);
YQL_ENSURE(ret, "Cannot cast " << TString{Name_}.Quote() << " constraint to " << T::Name());
- return ret;
- }
-
+ return ret;
+ }
+
std::string_view GetName() const {
- return Name_;
- }
-
-protected:
- ui64 Hash_;
+ return Name_;
+ }
+
+protected:
+ ui64 Hash_;
std::string_view Name_;
-};
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-class TConstraintSet {
-public:
- TConstraintSet() = default;
- TConstraintSet(const TConstraintSet&) = default;
- TConstraintSet(TConstraintSet&&) = default;
-
- TConstraintSet& operator =(const TConstraintSet&) = default;
- TConstraintSet& operator =(TConstraintSet&&) = default;
-
- template <class TConstraintType>
- const TConstraintType* GetConstraint() const {
- auto res = GetConstraint(TConstraintType::Name());
- return res ? res->template Cast<TConstraintType>() : nullptr;
- }
-
- template <class TConstraintType>
- const TConstraintType* RemoveConstraint() {
- auto res = RemoveConstraint(TConstraintType::Name());
- return res ? res->template Cast<TConstraintType>() : nullptr;
- }
-
- const TConstraintNode::TListType& GetAllConstraints() const {
- return Constraints_;
- }
-
- void Clear() {
- Constraints_.clear();
- }
-
- bool operator ==(const TConstraintSet& s) const {
- return Constraints_ == s.Constraints_;
- }
-
- bool operator !=(const TConstraintSet& s) const {
- return Constraints_ != s.Constraints_;
- }
-
+};
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class TConstraintSet {
+public:
+ TConstraintSet() = default;
+ TConstraintSet(const TConstraintSet&) = default;
+ TConstraintSet(TConstraintSet&&) = default;
+
+ TConstraintSet& operator =(const TConstraintSet&) = default;
+ TConstraintSet& operator =(TConstraintSet&&) = default;
+
+ template <class TConstraintType>
+ const TConstraintType* GetConstraint() const {
+ auto res = GetConstraint(TConstraintType::Name());
+ return res ? res->template Cast<TConstraintType>() : nullptr;
+ }
+
+ template <class TConstraintType>
+ const TConstraintType* RemoveConstraint() {
+ auto res = RemoveConstraint(TConstraintType::Name());
+ return res ? res->template Cast<TConstraintType>() : nullptr;
+ }
+
+ const TConstraintNode::TListType& GetAllConstraints() const {
+ return Constraints_;
+ }
+
+ void Clear() {
+ Constraints_.clear();
+ }
+
+ bool operator ==(const TConstraintSet& s) const {
+ return Constraints_ == s.Constraints_;
+ }
+
+ bool operator !=(const TConstraintSet& s) const {
+ return Constraints_ != s.Constraints_;
+ }
+
const TConstraintNode* GetConstraint(std::string_view name) const;
- void AddConstraint(const TConstraintNode* node);
+ void AddConstraint(const TConstraintNode* node);
const TConstraintNode* RemoveConstraint(std::string_view name);
-
+
void ToJson(NJson::TJsonWriter& writer) const;
-private:
- TConstraintNode::TListType Constraints_;
-};
-
-//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
-
-class TColumnSetConstraintNodeBase: public TConstraintNode {
-public:
- using TSetType = NSorted::TSimpleSet<TStringBuf>;
-
-protected:
- TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TSetType& columns);
- TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TVector<TStringBuf>& columns);
- TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TVector<TString>& columns);
- TColumnSetConstraintNodeBase(TColumnSetConstraintNodeBase&& constr);
-
-public:
- const TSetType& GetColumns() const {
- return Columns_;
- }
-
- bool Equals(const TConstraintNode& node) const override;
- bool Includes(const TConstraintNode& node) const override;
- void Out(IOutputStream& out) const override;
+private:
+ TConstraintNode::TListType Constraints_;
+};
+
+//////////////////////////////////////////////////////////////////////////////////////////////////////////////////////
+
+class TColumnSetConstraintNodeBase: public TConstraintNode {
+public:
+ using TSetType = NSorted::TSimpleSet<TStringBuf>;
+
+protected:
+ TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TSetType& columns);
+ TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TVector<TStringBuf>& columns);
+ TColumnSetConstraintNodeBase(TExprContext& ctx, TStringBuf name, const TVector<TString>& columns);
+ TColumnSetConstraintNodeBase(TColumnSetConstraintNodeBase&& constr);
+
+public:
+ const TSetType& GetColumns() const {
+ return Columns_;
+ }
+
+ bool Equals(const TConstraintNode& node) const override;
+ bool Includes(const TConstraintNode& node) const override;
+ void Out(IOutputStream& out) const override;
void ToJson(NJson::TJsonWriter& out) const override;
-
-protected:
- TSetType Columns_;
-};
-
+
+protected:
+ TSetType Columns_;
+};
+
class TSortedConstraintNode final: public TConstraintNode {
public:
using TContainerType = TSmallVec<std::pair<NSorted::TSimpleSet<std::string_view>, bool>>;
private:
- friend struct TExprContext;
-
+ friend struct TExprContext;
+
TSortedConstraintNode(TExprContext& ctx, TContainerType&& content);
- TSortedConstraintNode(TExprContext& ctx, const TSortedConstraintNode& constr, size_t prefixLength);
- TSortedConstraintNode(TSortedConstraintNode&& constr);
-public:
+ TSortedConstraintNode(TExprContext& ctx, const TSortedConstraintNode& constr, size_t prefixLength);
+ TSortedConstraintNode(TSortedConstraintNode&& constr);
+public:
static constexpr std::string_view Name() {
- return "Sorted";
- }
-
+ return "Sorted";
+ }
+
const TContainerType& GetContent() const {
return Content_;
- }
-
+ }
+
// TODO: deprecated, drop
const TVector<TStringBuf> GetColumns() const {
TVector<TStringBuf> result;
@@ -189,188 +189,188 @@ public:
return result;
}
- bool Equals(const TConstraintNode& node) const override;
- bool Includes(const TConstraintNode& node) const override;
- void Out(IOutputStream& out) const override;
+ bool Equals(const TConstraintNode& node) const override;
+ bool Includes(const TConstraintNode& node) const override;
+ void Out(IOutputStream& out) const override;
void ToJson(NJson::TJsonWriter& out) const override;
-
- bool IsPrefixOf(const TSortedConstraintNode& node) const;
- size_t GetCommonPrefixLength(const TSortedConstraintNode& node) const;
-
+
+ bool IsPrefixOf(const TSortedConstraintNode& node) const;
+ size_t GetCommonPrefixLength(const TSortedConstraintNode& node) const;
+
const TSortedConstraintNode* CutPrefix(size_t newPrefixLength, TExprContext& ctx) const;
- static const TSortedConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
+ static const TSortedConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
static const TSortedConstraintNode* FilterByType(const TSortedConstraintNode* sorted, const TStructExprType* outItemType, TExprContext& ctx);
-
-protected:
+
+protected:
TContainerType Content_;
-};
-
-class TGroupByConstraintNode final: public TColumnSetConstraintNodeBase {
-protected:
- friend struct TExprContext;
-
- TGroupByConstraintNode(TExprContext& ctx, const TVector<TStringBuf>& columns);
- TGroupByConstraintNode(TExprContext& ctx, const TVector<TString>& columns);
- TGroupByConstraintNode(TExprContext& ctx, const TGroupByConstraintNode& constr, size_t prefixLength);
- TGroupByConstraintNode(TGroupByConstraintNode&& constr);
-
- size_t GetCommonPrefixLength(const TGroupByConstraintNode& node) const;
-public:
+};
+
+class TGroupByConstraintNode final: public TColumnSetConstraintNodeBase {
+protected:
+ friend struct TExprContext;
+
+ TGroupByConstraintNode(TExprContext& ctx, const TVector<TStringBuf>& columns);
+ TGroupByConstraintNode(TExprContext& ctx, const TVector<TString>& columns);
+ TGroupByConstraintNode(TExprContext& ctx, const TGroupByConstraintNode& constr, size_t prefixLength);
+ TGroupByConstraintNode(TGroupByConstraintNode&& constr);
+
+ size_t GetCommonPrefixLength(const TGroupByConstraintNode& node) const;
+public:
static constexpr std::string_view Name() {
- return "GroupBy";
- }
-
- static const TGroupByConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
-};
-
-class TUniqueConstraintNode final: public TColumnSetConstraintNodeBase {
-protected:
- friend struct TExprContext;
-
- TUniqueConstraintNode(TExprContext& ctx, const TSetType& columns);
- TUniqueConstraintNode(TExprContext& ctx, const TVector<TStringBuf>& columns);
- TUniqueConstraintNode(TExprContext& ctx, const TVector<TString>& columns);
- TUniqueConstraintNode(TExprContext& ctx, const TStructExprType& itemType);
- TUniqueConstraintNode(TUniqueConstraintNode&& constr);
-
-public:
+ return "GroupBy";
+ }
+
+ static const TGroupByConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
+};
+
+class TUniqueConstraintNode final: public TColumnSetConstraintNodeBase {
+protected:
+ friend struct TExprContext;
+
+ TUniqueConstraintNode(TExprContext& ctx, const TSetType& columns);
+ TUniqueConstraintNode(TExprContext& ctx, const TVector<TStringBuf>& columns);
+ TUniqueConstraintNode(TExprContext& ctx, const TVector<TString>& columns);
+ TUniqueConstraintNode(TExprContext& ctx, const TStructExprType& itemType);
+ TUniqueConstraintNode(TUniqueConstraintNode&& constr);
+
+public:
static constexpr std::string_view Name() {
- return "Unique";
- }
-
- bool HasEqualColumns(TVector<TString> columns) const;
- bool HasEqualColumns(TVector<TStringBuf> columns) const;
-
- static const TUniqueConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
-};
-
-class TPassthroughConstraintNode final: public TConstraintNode {
-public:
+ return "Unique";
+ }
+
+ bool HasEqualColumns(TVector<TString> columns) const;
+ bool HasEqualColumns(TVector<TStringBuf> columns) const;
+
+ static const TUniqueConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
+};
+
+class TPassthroughConstraintNode final: public TConstraintNode {
+public:
using TKeyType = std::deque<std::string_view>;
using TPartType = NSorted::TSimpleMap<TKeyType, std::string_view>;
using TMapType = std::unordered_map<const TPassthroughConstraintNode*, TPartType>;
using TReverseMapType = NSorted::TSimpleMap<std::string_view, std::string_view>;
private:
- friend struct TExprContext;
-
- TPassthroughConstraintNode(TExprContext& ctx, const TStructExprType& itemType);
+ friend struct TExprContext;
+
+ TPassthroughConstraintNode(TExprContext& ctx, const TStructExprType& itemType);
TPassthroughConstraintNode(TExprContext& ctx, const TTupleExprType& itemType);
TPassthroughConstraintNode(TExprContext& ctx, const TMultiExprType& itemType);
- TPassthroughConstraintNode(TPassthroughConstraintNode&& constr);
+ TPassthroughConstraintNode(TPassthroughConstraintNode&& constr);
TPassthroughConstraintNode(TExprContext& ctx, TMapType&& mapping);
-public:
+public:
static constexpr std::string_view Name() {
- return "Passthrough";
- }
-
+ return "Passthrough";
+ }
+
const TMapType& GetColumnMapping() const;
TReverseMapType GetReverseMapping() const;
-
- bool Equals(const TConstraintNode& node) const override;
- bool Includes(const TConstraintNode& node) const override;
- void Out(IOutputStream& out) const override;
+
+ bool Equals(const TConstraintNode& node) const override;
+ bool Includes(const TConstraintNode& node) const override;
+ void Out(IOutputStream& out) const override;
void ToJson(NJson::TJsonWriter& out) const override;
-
+
const TPassthroughConstraintNode* ExtractField(TExprContext& ctx, const std::string_view& field) const;
- static const TPassthroughConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
-private:
- TMapType Mapping_;
-};
-
-class TEmptyConstraintNode final: public TConstraintNode {
-protected:
- friend struct TExprContext;
-
- TEmptyConstraintNode(TExprContext& ctx);
- TEmptyConstraintNode(TEmptyConstraintNode&& constr);
-
-public:
+ static const TPassthroughConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
+private:
+ TMapType Mapping_;
+};
+
+class TEmptyConstraintNode final: public TConstraintNode {
+protected:
+ friend struct TExprContext;
+
+ TEmptyConstraintNode(TExprContext& ctx);
+ TEmptyConstraintNode(TEmptyConstraintNode&& constr);
+
+public:
static constexpr std::string_view Name() {
- return "Empty";
- }
-
- bool Equals(const TConstraintNode& node) const override;
+ return "Empty";
+ }
+
+ bool Equals(const TConstraintNode& node) const override;
void ToJson(NJson::TJsonWriter& out) const override;
-
- static const TEmptyConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
-};
-
-class TVarIndexConstraintNode final: public TConstraintNode {
-public:
- using TMapType = NSorted::TSimpleMap<ui32, ui32>;
-
-protected:
- friend struct TExprContext;
-
- TVarIndexConstraintNode(TExprContext& ctx, const TMapType& mapping);
- TVarIndexConstraintNode(TExprContext& ctx, const TVariantExprType& itemType);
- TVarIndexConstraintNode(TExprContext& ctx, size_t mapItemsCount);
- TVarIndexConstraintNode(TVarIndexConstraintNode&& constr);
-
-public:
+
+ static const TEmptyConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
+};
+
+class TVarIndexConstraintNode final: public TConstraintNode {
+public:
+ using TMapType = NSorted::TSimpleMap<ui32, ui32>;
+
+protected:
+ friend struct TExprContext;
+
+ TVarIndexConstraintNode(TExprContext& ctx, const TMapType& mapping);
+ TVarIndexConstraintNode(TExprContext& ctx, const TVariantExprType& itemType);
+ TVarIndexConstraintNode(TExprContext& ctx, size_t mapItemsCount);
+ TVarIndexConstraintNode(TVarIndexConstraintNode&& constr);
+
+public:
static constexpr std::string_view Name() {
- return "VarIndex";
- }
-
- // multimap: result index -> {original indices}
- const TMapType& GetIndexMapping() const {
- return Mapping_;
- }
-
- // original index -> {result indices}
- TMapType GetReverseMapping() const;
-
- bool Equals(const TConstraintNode& node) const override;
- bool Includes(const TConstraintNode& node) const override;
- void Out(IOutputStream& out) const override;
+ return "VarIndex";
+ }
+
+ // multimap: result index -> {original indices}
+ const TMapType& GetIndexMapping() const {
+ return Mapping_;
+ }
+
+ // original index -> {result indices}
+ TMapType GetReverseMapping() const;
+
+ bool Equals(const TConstraintNode& node) const override;
+ bool Includes(const TConstraintNode& node) const override;
+ void Out(IOutputStream& out) const override;
void ToJson(NJson::TJsonWriter& out) const override;
-
- static const TVarIndexConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
-
-private:
- TMapType Mapping_;
-};
-
-class TMultiConstraintNode: public TConstraintNode {
-public:
- struct TConstraintKey {
- TStringBuf operator()(const TConstraintNode* node) const {
- return node->GetName();
- }
- };
- using TMapType = NSorted::TSimpleMap<ui32, TConstraintSet>;
-
-public:
- TMultiConstraintNode(TExprContext& ctx, const TMapType& items);
- TMultiConstraintNode(TExprContext& ctx, ui32 index, const TConstraintSet& constraints);
- TMultiConstraintNode(TMultiConstraintNode&& constr);
-
-public:
+
+ static const TVarIndexConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
+
+private:
+ TMapType Mapping_;
+};
+
+class TMultiConstraintNode: public TConstraintNode {
+public:
+ struct TConstraintKey {
+ TStringBuf operator()(const TConstraintNode* node) const {
+ return node->GetName();
+ }
+ };
+ using TMapType = NSorted::TSimpleMap<ui32, TConstraintSet>;
+
+public:
+ TMultiConstraintNode(TExprContext& ctx, const TMapType& items);
+ TMultiConstraintNode(TExprContext& ctx, ui32 index, const TConstraintSet& constraints);
+ TMultiConstraintNode(TMultiConstraintNode&& constr);
+
+public:
static constexpr std::string_view Name() {
- return "Multi";
- }
-
- const TMapType& GetItems() const {
- return Items_;
- }
-
- const TConstraintSet* GetItem(ui32 index) const {
- return Items_.FindPtr(index);
- }
-
- bool Equals(const TConstraintNode& node) const override;
- bool Includes(const TConstraintNode& node) const override;
- void Out(IOutputStream& out) const override;
+ return "Multi";
+ }
+
+ const TMapType& GetItems() const {
+ return Items_;
+ }
+
+ const TConstraintSet* GetItem(ui32 index) const {
+ return Items_.FindPtr(index);
+ }
+
+ bool Equals(const TConstraintNode& node) const override;
+ bool Includes(const TConstraintNode& node) const override;
+ void Out(IOutputStream& out) const override;
void ToJson(NJson::TJsonWriter& out) const override;
-
- static const TMultiConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
-
+
+ static const TMultiConstraintNode* MakeCommon(const TVector<const TConstraintSet*>& constraints, TExprContext& ctx);
+
bool FilteredIncludes(const TConstraintNode& node, const THashSet<TString>& blacklist) const;
-protected:
- TMapType Items_;
-};
-
-} // namespace NYql
-
+protected:
+ TMapType Items_;
+};
+
+} // namespace NYql
+
diff --git a/ydb/library/yql/ast/yql_expr.cpp b/ydb/library/yql/ast/yql_expr.cpp
index 82f6c5812a..1d397538b9 100644
--- a/ydb/library/yql/ast/yql_expr.cpp
+++ b/ydb/library/yql/ast/yql_expr.cpp
@@ -6,7 +6,7 @@
#include <library/cpp/containers/stack_vector/stack_vec.h>
#include <util/generic/hash.h>
-#include <util/generic/size_literals.h>
+#include <util/generic/size_literals.h>
#include <util/string/cast.h>
#include <util/string/join.h>
@@ -50,8 +50,8 @@ namespace {
out << " <" << node.Content() << ">";
}
- if (node.GetTypeAnn()) {
- out << ", type: " << *node.GetTypeAnn();
+ if (node.GetTypeAnn()) {
+ out << ", type: " << *node.GetTypeAnn();
}
if (const auto scope = node.GetDependencyScope()) {
@@ -117,7 +117,7 @@ namespace {
TExprNode::TPtr&& ProcessNode(const TAstNode& node, TExprNode::TPtr&& exprNode) {
if (TypeAnnotationIndex != Max<ui32>()) {
- exprNode->SetTypeAnn(CompileTypeAnnotation(node));
+ exprNode->SetTypeAnn(CompileTypeAnnotation(node));
}
return std::move(exprNode);
@@ -181,12 +181,12 @@ namespace {
else if (node.GetContent() == TStringBuf("Generic")) {
return Expr.MakeType<TGenericExprType>();
}
- else if (node.GetContent() == TStringBuf("EmptyList")) {
- return Expr.MakeType<TEmptyListExprType>();
- }
- else if (node.GetContent() == TStringBuf("EmptyDict")) {
- return Expr.MakeType<TEmptyDictExprType>();
- }
+ else if (node.GetContent() == TStringBuf("EmptyList")) {
+ return Expr.MakeType<TEmptyListExprType>();
+ }
+ else if (node.GetContent() == TStringBuf("EmptyDict")) {
+ return Expr.MakeType<TEmptyDictExprType>();
+ }
else {
AddError(node, TStringBuilder() << "Unknown type annotation: " << node.GetContent());
return nullptr;
@@ -477,12 +477,12 @@ namespace {
return nullptr;
TString tag(node.GetChild(2)->GetContent());
- auto ann = Expr.MakeType<TTaggedExprType>(type, tag);
- if (!ann->Validate(node.GetPosition(), Expr)) {
- return nullptr;
- }
-
- return ann;
+ auto ann = Expr.MakeType<TTaggedExprType>(type, tag);
+ if (!ann->Validate(node.GetPosition(), Expr)) {
+ return nullptr;
+ }
+
+ return ann;
} else if (content == TStringBuf("Error")) {
if (node.GetChildrenCount() != 5 || !node.GetChild(1)->IsAtom() ||
!node.GetChild(2)->IsAtom() || !node.GetChild(3)->IsAtom() || !node.GetChild(4)->IsAtom()) {
@@ -775,13 +775,13 @@ namespace {
}
case ETypeAnnotationKind::EmptyList:
- {
- return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("EmptyList"), pool);
- }
+ {
+ return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("EmptyList"), pool);
+ }
case ETypeAnnotationKind::EmptyDict:
- {
- return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("EmptyDict"), pool);
- }
+ {
+ return TAstNode::NewLiteralAtom(TPosition(), TStringBuf("EmptyDict"), pool);
+ }
case ETypeAnnotationKind::LastType:
YQL_ENSURE(false, "Unknown kind: " << annotation.GetKind());
@@ -800,8 +800,8 @@ namespace {
if ((flags & TExprAnnotationFlags::Types)) {
TAstNode* typeAnn = nullptr;
if (exprNode) {
- YQL_ENSURE(exprNode->GetTypeAnn());
- typeAnn = ConvertTypeAnnotationToAst(*exprNode->GetTypeAnn(), pool, refAtoms);
+ YQL_ENSURE(exprNode->GetTypeAnn());
+ typeAnn = ConvertTypeAnnotationToAst(*exprNode->GetTypeAnn(), pool, refAtoms);
} else {
typeAnn = TAstNode::NewLiteralAtom(node->GetPosition(), TStringBuf("."), pool);
}
@@ -1972,7 +1972,7 @@ bool CompileExpr(TAstNode& astRoot, TExprNode::TPtr& exprRoot, TExprContext& ctx
compileCtx.PushFrame();
auto world = compileCtx.Expr.NewWorld(astRoot.GetPosition());
if (typeAnnotationIndex != Max<ui32>()) {
- world->SetTypeAnn(compileCtx.Expr.MakeType<TWorldExprType>());
+ world->SetTypeAnn(compileCtx.Expr.MakeType<TWorldExprType>());
}
compileCtx.Frames.back().Bindings[TStringBuf("world")] = {std::move(world)};
@@ -2047,56 +2047,56 @@ ui64 CalcBloom(const ui64 id) {
}
inline bool InBloom(const ui64 set, const ui64 bloom) {
- return (bloom >> 1) == ((bloom & set) >> 1);
+ return (bloom >> 1) == ((bloom & set) >> 1);
}
-EChangeState GetChanges(TExprNode* start, const TNodeOnNodeOwnedMap& replaces, const TNodeMap<TNodeOnNodeOwnedMap>& localReplaces,
- TNodeMap<EChangeState>& changes, TNodeMap<bool>& updatedLambdas);
-
-EChangeState DoGetChanges(TExprNode* start, const TNodeOnNodeOwnedMap& replaces, const TNodeMap<TNodeOnNodeOwnedMap>& localReplaces,
+EChangeState GetChanges(TExprNode* start, const TNodeOnNodeOwnedMap& replaces, const TNodeMap<TNodeOnNodeOwnedMap>& localReplaces,
+ TNodeMap<EChangeState>& changes, TNodeMap<bool>& updatedLambdas);
+
+EChangeState DoGetChanges(TExprNode* start, const TNodeOnNodeOwnedMap& replaces, const TNodeMap<TNodeOnNodeOwnedMap>& localReplaces,
TNodeMap<EChangeState>& changes, TNodeMap<bool>& updatedLambdas) {
-
- if (start->GetBloom() & 1ULL) {
- bool maybe = false;
+
+ if (start->GetBloom() & 1ULL) {
+ bool maybe = false;
for (const auto& repl : replaces) {
if (repl.second && !repl.first->Dead()) {
- if (TExprNode::Argument != repl.first->Type()) {
- maybe = true;
- break;
- }
-
+ if (TExprNode::Argument != repl.first->Type()) {
+ maybe = true;
+ break;
+ }
+
if (!repl.first->GetBloom())
const_cast<TExprNode*>(repl.first)->SetBloom(CalcBloom(repl.first->UniqueId()));
-
+
if (InBloom(start->GetBloom(), repl.first->GetBloom())) {
- maybe = true;
- break;
- }
- }
- }
-
- if (!maybe) {
- return EChangeState::Unchanged;
- }
- }
-
+ maybe = true;
+ break;
+ }
+ }
+ }
+
+ if (!maybe) {
+ return EChangeState::Unchanged;
+ }
+ }
+
start->SetBloom(1ULL);
- ui32 combinedState = EChangeState::Unchanged;
- bool incompleteBloom = false;
- start->ForEachChild([&](TExprNode& child) {
- combinedState |= GetChanges(&child, replaces, localReplaces, changes, updatedLambdas);
+ ui32 combinedState = EChangeState::Unchanged;
+ bool incompleteBloom = false;
+ start->ForEachChild([&](TExprNode& child) {
+ combinedState |= GetChanges(&child, replaces, localReplaces, changes, updatedLambdas);
start->SetBloom(start->GetBloom() | child.GetBloom());
- incompleteBloom = incompleteBloom || (child.Type() != TExprNode::Arguments && !child.GetBloom());
- });
+ incompleteBloom = incompleteBloom || (child.Type() != TExprNode::Arguments && !child.GetBloom());
+ });
if (incompleteBloom) {
start->SetBloom(0ULL);
- }
-
- return (EChangeState)combinedState;
-}
-
-EChangeState GetChanges(TExprNode* start, const TNodeOnNodeOwnedMap& replaces, const TNodeMap<TNodeOnNodeOwnedMap>& localReplaces,
- TNodeMap<EChangeState>& changes, TNodeMap<bool>& updatedLambdas) {
+ }
+
+ return (EChangeState)combinedState;
+}
+
+EChangeState GetChanges(TExprNode* start, const TNodeOnNodeOwnedMap& replaces, const TNodeMap<TNodeOnNodeOwnedMap>& localReplaces,
+ TNodeMap<EChangeState>& changes, TNodeMap<bool>& updatedLambdas) {
if (start->Type() == TExprNode::Arguments) {
return EChangeState::Unchanged;
}
@@ -2118,32 +2118,32 @@ EChangeState GetChanges(TExprNode* start, const TNodeOnNodeOwnedMap& replaces, c
return state = EChangeState::Unchanged;
}
- if (start->Type() == TExprNode::Lambda) {
- TNodeOnNodeOwnedMap newReplaces = replaces;
+ if (start->Type() == TExprNode::Lambda) {
+ TNodeOnNodeOwnedMap newReplaces = replaces;
start->Head().ForEachChild([&](const TExprNode& arg){ newReplaces[&arg] = {}; });
const auto locIt = localReplaces.find(start);
- if (locIt != localReplaces.end()) {
- for (auto& r: locIt->second) {
- newReplaces[r.first] = r.second;
+ if (locIt != localReplaces.end()) {
+ for (auto& r: locIt->second) {
+ newReplaces[r.first] = r.second;
}
}
- state = DoGetChanges(start, newReplaces, localReplaces, changes, updatedLambdas);
-
- if ((state & EChangeState::Changed) != 0) {
- updatedLambdas.emplace(start, false);
+ state = DoGetChanges(start, newReplaces, localReplaces, changes, updatedLambdas);
+
+ if ((state & EChangeState::Changed) != 0) {
+ updatedLambdas.emplace(start, false);
}
- return state;
+ return state;
}
- return state = DoGetChanges(start, replaces, localReplaces, changes, updatedLambdas);
+ return state = DoGetChanges(start, replaces, localReplaces, changes, updatedLambdas);
}
template<bool KeepTypeAnns>
-TExprNode::TPtr DoReplace(const TExprNode::TPtr& start, const TNodeOnNodeOwnedMap& replaces, const TNodeMap<TNodeOnNodeOwnedMap>& localReplaces,
+TExprNode::TPtr DoReplace(const TExprNode::TPtr& start, const TNodeOnNodeOwnedMap& replaces, const TNodeMap<TNodeOnNodeOwnedMap>& localReplaces,
TNodeMap<EChangeState>& changes, TNodeOnNodeOwnedMap& processed, TExprContext& ctx) {
auto& target = processed[start.Get()];
@@ -2162,19 +2162,19 @@ TExprNode::TPtr DoReplace(const TExprNode::TPtr& start, const TNodeOnNodeOwnedMa
const bool isChanged = (changeIt->second & EChangeState::Changed) != 0;
if (isChanged) {
if (start->Type() == TExprNode::Lambda) {
- TNodeOnNodeOwnedMap newReplaces = replaces;
+ TNodeOnNodeOwnedMap newReplaces = replaces;
const auto locIt = localReplaces.find(start.Get());
- YQL_ENSURE(locIt != localReplaces.end(), "Missing local changes");
- for (auto& r: locIt->second) {
- newReplaces[r.first] = r.second;
- }
-
+ YQL_ENSURE(locIt != localReplaces.end(), "Missing local changes");
+ for (auto& r: locIt->second) {
+ newReplaces[r.first] = r.second;
+ }
+
const auto& args = start->Head();
TExprNode::TListType newArgsList;
newArgsList.reserve(args.ChildrenSize());
args.ForEachChild([&](const TExprNode& arg) {
const auto argIt = newReplaces.find(&arg);
- YQL_ENSURE(argIt != newReplaces.end(), "Missing argument");
+ YQL_ENSURE(argIt != newReplaces.end(), "Missing argument");
processed.emplace(&arg, argIt->second);
newArgsList.emplace_back(argIt->second);
});
@@ -2226,10 +2226,10 @@ void EnsureNoBadReplaces(const TExprNode& start, const TNodeOnNodeOwnedMap& repl
}
if (start.Type() == TExprNode::Lambda) {
- TNodeOnNodeOwnedMap newReplaces = replaces;
+ TNodeOnNodeOwnedMap newReplaces = replaces;
start.Head().ForEachChild([&](const TExprNode& arg){ newReplaces[&arg] = {}; });
start.ForEachChild([&](const TExprNode& child){ EnsureNoBadReplaces(child, newReplaces, std::move(visited)); });
- } else {
+ } else {
start.ForEachChild([&](const TExprNode& child){ EnsureNoBadReplaces(child, replaces, std::move(visited)); });
}
}
@@ -2251,8 +2251,8 @@ TExprNode::TPtr ReplaceNodesImpl(TExprNode::TPtr&& start, const TNodeOnNodeOwned
TNodeMap<EChangeState> changes;
TNodeMap<bool> updatedLambdas;
- TNodeMap<TNodeOnNodeOwnedMap> localReplaces;
- if ((GetChanges(start.Get(), replaces, localReplaces, changes, updatedLambdas) & EChangeState::Changed) == 0) {
+ TNodeMap<TNodeOnNodeOwnedMap> localReplaces;
+ if ((GetChanges(start.Get(), replaces, localReplaces, changes, updatedLambdas) & EChangeState::Changed) == 0) {
return std::move(start);
}
@@ -2261,7 +2261,7 @@ TExprNode::TPtr ReplaceNodesImpl(TExprNode::TPtr&& start, const TNodeOnNodeOwned
changes.clear();
for (auto& x : updatedLambdas) {
if (!x.second) {
- TNodeOnNodeOwnedMap& lambdaReplaces = localReplaces[x.first];
+ TNodeOnNodeOwnedMap& lambdaReplaces = localReplaces[x.first];
const auto& args = x.first->Head();
args.ForEachChild([&](const TExprNode& arg) {
const auto newArg = lambdaReplaces.emplace(&arg, ctx.ShallowCopy(arg)).first->second;
@@ -2273,7 +2273,7 @@ TExprNode::TPtr ReplaceNodesImpl(TExprNode::TPtr&& start, const TNodeOnNodeOwned
}
auto prevSize = updatedLambdas.size();
- GetChanges(start.Get(), replaces, localReplaces, changes, updatedLambdas);
+ GetChanges(start.Get(), replaces, localReplaces, changes, updatedLambdas);
if (updatedLambdas.size() == prevSize) {
break;
}
@@ -2458,8 +2458,8 @@ TExprNode::TPtr TExprContext::ChangeChild(const TExprNode& node, ui32 index, TEx
TExprNode::TPtr TExprContext::ExactChangeChildren(const TExprNode& node, TExprNode::TListType&& children) {
const auto newNode = node.ChangeChildren(AllocateNextUniqueId(), std::move(children));
- newNode->SetTypeAnn(node.GetTypeAnn());
- newNode->CopyConstraints(node);
+ newNode->SetTypeAnn(node.GetTypeAnn());
+ newNode->CopyConstraints(node);
newNode->SetState(node.GetState());
newNode->Result = node.Result;
ExprNodes.emplace_back(newNode.Get());
@@ -2469,8 +2469,8 @@ TExprNode::TPtr TExprContext::ExactChangeChildren(const TExprNode& node, TExprNo
TExprNode::TPtr TExprContext::ExactShallowCopy(const TExprNode& node) {
YQL_ENSURE(node.Type() != TExprNode::Lambda);
const auto newNode = node.Clone(AllocateNextUniqueId());
- newNode->SetTypeAnn(node.GetTypeAnn());
- newNode->CopyConstraints(node);
+ newNode->SetTypeAnn(node.GetTypeAnn());
+ newNode->CopyConstraints(node);
newNode->SetState(node.GetState());
newNode->Result = node.Result;
ExprNodes.emplace_back(newNode.Get());
@@ -2566,7 +2566,7 @@ TExprNode::TPtr TExprContext::FuseLambdas(const TExprNode& outer, const TExprNod
}
TExprNode::TPtr TExprContext::DeepCopy(const TExprNode& node, TExprContext& nodeCtx, TNodeOnNodeOwnedMap& deepClones,
- bool internStrings, bool copyTypes, bool copyResult, TCustomDeepCopier customCopier)
+ bool internStrings, bool copyTypes, bool copyResult, TCustomDeepCopier customCopier)
{
const auto ins = deepClones.emplace(&node, nullptr);
if (ins.second) {
@@ -2576,7 +2576,7 @@ TExprNode::TPtr TExprContext::DeepCopy(const TExprNode& node, TExprContext& node
if (customCopier && customCopier(node, children)) {
} else {
node.ForEachChild([&](const TExprNode& child) {
- children.emplace_back(DeepCopy(child, nodeCtx, deepClones, internStrings, copyTypes, copyResult, customCopier));
+ children.emplace_back(DeepCopy(child, nodeCtx, deepClones, internStrings, copyTypes, copyResult, customCopier));
});
}
@@ -2589,10 +2589,10 @@ TExprNode::TPtr TExprContext::DeepCopy(const TExprNode& node, TExprContext& node
newNode->SetTypeAnn(node.GetTypeAnn());
}
- if (copyResult && node.IsCallable() && node.HasResult()) {
- newNode->SetResult(nodeCtx.ShallowCopy(node.GetResult()));
- }
-
+ if (copyResult && node.IsCallable() && node.HasResult()) {
+ newNode->SetResult(nodeCtx.ShallowCopy(node.GetResult()));
+ }
+
ins.first->second = newNode;
ExprNodes.emplace_back(ins.first->second.Get());
}
@@ -2631,10 +2631,10 @@ TNodeException::TNodeException(const TPositionHandle& pos)
{
}
-bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprContext& ctx) {
+bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprContext& ctx) {
if (name.empty()) {
- ctx.AddError(TIssue(position,
- TStringBuilder() << "Empty " << descr << " name is not allowed"));
+ ctx.AddError(TIssue(position,
+ TStringBuilder() << "Empty " << descr << " name is not allowed"));
return false;
}
@@ -2644,17 +2644,17 @@ bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprCo
return false;
}
- if (name.size() > 16_KB) {
- ctx.AddError(TIssue(position, TStringBuilder() <<
+ if (name.size() > 16_KB) {
+ ctx.AddError(TIssue(position, TStringBuilder() <<
TString(descr).to_title() << " name length must be less than " << 16_KB));
- return false;
- }
-
+ return false;
+ }
+
return true;
}
-bool ValidateName(TPositionHandle position, TStringBuf name, TStringBuf descr, TExprContext& ctx) {
- return ValidateName(ctx.GetPosition(position), name, descr, ctx);
+bool ValidateName(TPositionHandle position, TStringBuf name, TStringBuf descr, TExprContext& ctx) {
+ return ValidateName(ctx.GetPosition(position), name, descr, ctx);
}
bool TDataExprParamsType::Validate(TPosition position, TExprContext& ctx) const {
@@ -2685,7 +2685,7 @@ bool TDataExprParamsType::Validate(TPositionHandle position, TExprContext& ctx)
}
bool TItemExprType::Validate(TPosition position, TExprContext& ctx) const {
- return ValidateName(position, Name, "member", ctx);
+ return ValidateName(position, Name, "member", ctx);
}
bool TItemExprType::Validate(TPositionHandle position, TExprContext& ctx) const {
@@ -2853,14 +2853,14 @@ bool TCallableExprType::Validate(TPositionHandle position, TExprContext& ctx) co
return Validate(ctx.GetPosition(position), ctx);
}
-bool TTaggedExprType::Validate(TPosition position, TExprContext& ctx) const {
- return ValidateName(position, Tag, "tag", ctx);
-}
-
-bool TTaggedExprType::Validate(TPositionHandle position, TExprContext& ctx) const {
- return Validate(ctx.GetPosition(position), ctx);
-}
-
+bool TTaggedExprType::Validate(TPosition position, TExprContext& ctx) const {
+ return ValidateName(position, Tag, "tag", ctx);
+}
+
+bool TTaggedExprType::Validate(TPositionHandle position, TExprContext& ctx) const {
+ return Validate(ctx.GetPosition(position), ctx);
+}
+
TExprContext::TExprContext(ui64 nextUniqueId)
: StringPool(4096)
, NextUniqueId(nextUniqueId)
@@ -2876,7 +2876,7 @@ TExprContext::TExprContext(ui64 nextUniqueId)
IssueManager.SetWarningToErrorTreatMessage(
"Treat warning as error mode enabled. "
"To disable it use \"pragma warning(\"default\", <code>);\"");
- IssueManager.SetIssueCountLimit(100);
+ IssueManager.SetIssueCountLimit(100);
}
TPositionHandle TExprContext::AppendPosition(const TPosition& pos) {
diff --git a/ydb/library/yql/ast/yql_expr.h b/ydb/library/yql/ast/yql_expr.h
index d6eb544f9a..db4cddaf5c 100644
--- a/ydb/library/yql/ast/yql_expr.h
+++ b/ydb/library/yql/ast/yql_expr.h
@@ -5,7 +5,7 @@
#include "yql_type_string.h"
#include "yql_expr_builder.h"
#include "yql_gc_nodes.h"
-#include "yql_constraint.h"
+#include "yql_constraint.h"
#include "yql_pos_handle.h"
#include <ydb/library/yql/utils/yql_panic.h>
@@ -27,7 +27,7 @@
#include <util/generic/algorithm.h>
#include <util/digest/murmur.h>
-#include <algorithm>
+#include <algorithm>
#include <unordered_set>
#include <unordered_map>
#include <span>
@@ -393,8 +393,8 @@ private:
struct TExprContext;
-bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprContext& ctx);
-bool ValidateName(TPositionHandle position, TStringBuf name, TStringBuf descr, TExprContext& ctx);
+bool ValidateName(TPosition position, TStringBuf name, TStringBuf descr, TExprContext& ctx);
+bool ValidateName(TPositionHandle position, TStringBuf name, TStringBuf descr, TExprContext& ctx);
class TItemExprType : public TTypeAnnotationNode {
public:
@@ -1088,9 +1088,9 @@ public:
return Tag == other.Tag && GetBaseType() == other.GetBaseType();
}
- bool Validate(TPosition position, TExprContext& ctx) const;
- bool Validate(TPositionHandle position, TExprContext& ctx) const;
-
+ bool Validate(TPosition position, TExprContext& ctx) const;
+ bool Validate(TPositionHandle position, TExprContext& ctx) const;
+
private:
const TTypeAnnotationNode* BaseType;
const TStringBuf Tag;
@@ -1348,13 +1348,13 @@ public:
xx(TypeInProgress, 1) \
xx(TypePending, 2) \
xx(TypeComplete, 3) \
- xx(ConstrInProgress, 4) \
- xx(ConstrPending, 5) \
- xx(ConstrComplete, 6) \
- xx(ExecutionRequired, 7) \
- xx(ExecutionInProgress, 8) \
- xx(ExecutionPending, 9) \
- xx(ExecutionComplete, 10) \
+ xx(ConstrInProgress, 4) \
+ xx(ConstrPending, 5) \
+ xx(ConstrComplete, 6) \
+ xx(ExecutionRequired, 7) \
+ xx(ExecutionInProgress, 8) \
+ xx(ExecutionPending, 9) \
+ xx(ExecutionComplete, 10) \
xx(Error, 11) \
xx(Last, 12)
@@ -1392,12 +1392,12 @@ public:
return Type() == TExprNode::Callable && names.end() != std::find(names.begin(), names.end(), Content());
}
- template <class TKey>
+ template <class TKey>
bool IsCallable(const THashSet<TKey>& names) const {
ENSURE_NOT_DELETED
return Type() == TExprNode::Callable && names.contains(Content());
- }
-
+ }
+
bool IsCallable() const {
ENSURE_NOT_DELETED
return Type() == TExprNode::Callable;
@@ -1435,22 +1435,22 @@ public:
bool IsComposable() const {
ENSURE_NOT_DELETED
- return !IsLambda() && TypeAnnotation_->IsComposable();
+ return !IsLambda() && TypeAnnotation_->IsComposable();
}
bool IsPersistable() const {
ENSURE_NOT_DELETED
- return !IsLambda() && TypeAnnotation_->IsPersistable();
+ return !IsLambda() && TypeAnnotation_->IsPersistable();
}
bool IsComputable() const {
ENSURE_NOT_DELETED
- return !IsLambda() && TypeAnnotation_->IsComputable();
+ return !IsLambda() && TypeAnnotation_->IsComputable();
}
bool IsInspectable() const {
ENSURE_NOT_DELETED
- return !IsLambda() && TypeAnnotation_->IsInspectable();
+ return !IsLambda() && TypeAnnotation_->IsInspectable();
}
bool ForDisclosing() const {
@@ -1502,7 +1502,7 @@ public:
if (!--RefCount_) {
Result.Reset();
Children_.clear();
- Constraints_.Clear();
+ Constraints_.Clear();
MarkDead();
}
}
@@ -1647,56 +1647,56 @@ public:
return UniqueId_;
}
- const TConstraintNode* GetConstraint(TStringBuf name) const {
- ENSURE_NOT_DELETED
- Y_ENSURE(static_cast<EState>(State) >= EState::ConstrComplete);
- return Constraints_.GetConstraint(name);
- }
-
- template <class TConstraintType>
- const TConstraintType* GetConstraint() const {
- ENSURE_NOT_DELETED
- Y_ENSURE(static_cast<EState>(State) >= EState::ConstrComplete);
- return Constraints_.GetConstraint<TConstraintType>();
- }
-
- const TConstraintNode::TListType& GetAllConstraints() const {
- ENSURE_NOT_DELETED
- Y_ENSURE(static_cast<EState>(State) >= EState::ConstrComplete);
- return Constraints_.GetAllConstraints();
- }
-
- const TConstraintSet& GetConstraintSet() const {
- ENSURE_NOT_DELETED
- Y_ENSURE(static_cast<EState>(State) >= EState::ConstrComplete);
- return Constraints_;
- }
-
- void AddConstraint(const TConstraintNode* node) {
- ENSURE_NOT_DELETED
- ENSURE_NOT_FROZEN
- Y_ENSURE(static_cast<EState>(State) >= EState::TypeComplete);
- Y_ENSURE(!StartsExecution());
- Constraints_.AddConstraint(node);
- State = EState::ConstrComplete;
- }
-
- void CopyConstraints(const TExprNode& node) {
- ENSURE_NOT_DELETED
- ENSURE_NOT_FROZEN
- Y_ENSURE(static_cast<EState>(State) >= EState::TypeComplete);
- Constraints_ = node.Constraints_;
- State = EState::ConstrComplete;
- }
-
- void SetConstraints(const TConstraintSet& constraints) {
- ENSURE_NOT_DELETED
- ENSURE_NOT_FROZEN
- Y_ENSURE(static_cast<EState>(State) >= EState::TypeComplete);
- Constraints_ = constraints;
- State = EState::ConstrComplete;
- }
-
+ const TConstraintNode* GetConstraint(TStringBuf name) const {
+ ENSURE_NOT_DELETED
+ Y_ENSURE(static_cast<EState>(State) >= EState::ConstrComplete);
+ return Constraints_.GetConstraint(name);
+ }
+
+ template <class TConstraintType>
+ const TConstraintType* GetConstraint() const {
+ ENSURE_NOT_DELETED
+ Y_ENSURE(static_cast<EState>(State) >= EState::ConstrComplete);
+ return Constraints_.GetConstraint<TConstraintType>();
+ }
+
+ const TConstraintNode::TListType& GetAllConstraints() const {
+ ENSURE_NOT_DELETED
+ Y_ENSURE(static_cast<EState>(State) >= EState::ConstrComplete);
+ return Constraints_.GetAllConstraints();
+ }
+
+ const TConstraintSet& GetConstraintSet() const {
+ ENSURE_NOT_DELETED
+ Y_ENSURE(static_cast<EState>(State) >= EState::ConstrComplete);
+ return Constraints_;
+ }
+
+ void AddConstraint(const TConstraintNode* node) {
+ ENSURE_NOT_DELETED
+ ENSURE_NOT_FROZEN
+ Y_ENSURE(static_cast<EState>(State) >= EState::TypeComplete);
+ Y_ENSURE(!StartsExecution());
+ Constraints_.AddConstraint(node);
+ State = EState::ConstrComplete;
+ }
+
+ void CopyConstraints(const TExprNode& node) {
+ ENSURE_NOT_DELETED
+ ENSURE_NOT_FROZEN
+ Y_ENSURE(static_cast<EState>(State) >= EState::TypeComplete);
+ Constraints_ = node.Constraints_;
+ State = EState::ConstrComplete;
+ }
+
+ void SetConstraints(const TConstraintSet& constraints) {
+ ENSURE_NOT_DELETED
+ ENSURE_NOT_FROZEN
+ Y_ENSURE(static_cast<EState>(State) >= EState::TypeComplete);
+ Constraints_ = constraints;
+ State = EState::ConstrComplete;
+ }
+
static TPtr NewAtom(ui64 uniqueId, TPositionHandle pos, const TStringBuf& content, ui32 flags) {
return Make(pos, Atom, {}, content, flags, uniqueId);
}
@@ -1769,15 +1769,15 @@ public:
return Make(Position_, (EType)Type_, std::move(newChildren), Content(), Flags_, newUniqueId);
}
- void SetTypeAnn(const TTypeAnnotationNode* typeAnn) {
- TypeAnnotation_ = typeAnn;
- State = TypeAnnotation_ ? EState::TypeComplete : EState::Initial;
- }
-
- const TTypeAnnotationNode* GetTypeAnn() const {
- return TypeAnnotation_;
- }
-
+ void SetTypeAnn(const TTypeAnnotationNode* typeAnn) {
+ TypeAnnotation_ = typeAnn;
+ State = TypeAnnotation_ ? EState::TypeComplete : EState::Initial;
+ }
+
+ const TTypeAnnotationNode* GetTypeAnn() const {
+ return TypeAnnotation_;
+ }
+
EState GetState() const {
return State;
}
@@ -1929,7 +1929,7 @@ private:
}
TListType Children_;
- TConstraintSet Constraints_;
+ TConstraintSet Constraints_;
const char* Content_ = nullptr;
@@ -2032,37 +2032,37 @@ struct TExprStep {
Configure,
ExprEval,
DiscoveryIO,
- Epochs,
+ Epochs,
Intents,
LoadTablesMetadata,
- RewriteIO,
- Recapture,
- LastLevel
+ RewriteIO,
+ Recapture,
+ LastLevel
};
- TExprStep()
+ TExprStep()
{
}
void Done(ELevel level) {
- Steps_.Set(level);
- }
-
- void Reset() {
- Steps_.Reset();
+ Steps_.Set(level);
}
- TExprStep& Repeat(ELevel level) {
- Steps_.Reset(level);
- return *this;
+ void Reset() {
+ Steps_.Reset();
}
+ TExprStep& Repeat(ELevel level) {
+ Steps_.Reset(level);
+ return *this;
+ }
+
bool IsDone(ELevel level) {
- return Steps_.Test(level);
+ return Steps_.Test(level);
}
private:
- TEnumBitSet<ELevel, ExpandApplyForLambdas, LastLevel> Steps_;
+ TEnumBitSet<ELevel, ExpandApplyForLambdas, LastLevel> Steps_;
};
template <typename T>
@@ -2240,14 +2240,14 @@ struct TExprContext : private TNonCopyable {
std::unordered_set<std::string_view> Strings;
std::stack<std::unique_ptr<const TTypeAnnotationNode>> TypeNodes;
- std::stack<std::unique_ptr<const TConstraintNode>> ConstraintNodes;
+ std::stack<std::unique_ptr<const TConstraintNode>> ConstraintNodes;
std::deque<std::unique_ptr<TExprNode>> ExprNodes;
TSingletonTypeCache SingletonTypeCache;
std::unordered_set<const TTypeAnnotationNode*, TTypeAnnotationNode::THash, TTypeAnnotationNode::TEqual> TypeSet;
- std::unordered_set<const TConstraintNode*, TConstraintNode::THash, TConstraintNode::TEqual> ConstraintSet;
+ std::unordered_set<const TConstraintNode*, TConstraintNode::THash, TConstraintNode::TEqual> ConstraintSet;
std::unordered_map<const TTypeAnnotationNode*, TExprNode::TPtr> TypeAsNodeCache;
- std::unordered_set<TStringBuf, THash<TStringBuf>> DisabledConstraints;
+ std::unordered_set<TStringBuf, THash<TStringBuf>> DisabledConstraints;
ui64 NextUniqueId = 0;
ui64 NodeAllocationCounter = 0;
@@ -2296,32 +2296,32 @@ struct TExprContext : private TNonCopyable {
return TExprNodeBuilder(pos, *this);
}
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr RenameNode(const TExprNode& node, const TStringBuf& name);
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr ShallowCopy(const TExprNode& node);
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr ChangeChildren(const TExprNode& node, TExprNode::TListType&& children);
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr ChangeChild(const TExprNode& node, ui32 index, TExprNode::TPtr&& child);
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr ExactChangeChildren(const TExprNode& node, TExprNode::TListType&& children);
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr ExactShallowCopy(const TExprNode& node);
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr DeepCopyLambda(const TExprNode& node, TExprNode::TListType&& body);
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr DeepCopyLambda(const TExprNode& node, TExprNode::TPtr&& body = TExprNode::TPtr());
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr FuseLambdas(const TExprNode& outer, const TExprNode& inner);
using TCustomDeepCopier = std::function<bool(const TExprNode& node, TExprNode::TListType& newChildren)>;
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr DeepCopy(const TExprNode& node, TExprContext& nodeContext, TNodeOnNodeOwnedMap& deepClones,
- bool internStrings, bool copyTypes, bool copyResult = false, TCustomDeepCopier customCopier = {});
+ bool internStrings, bool copyTypes, bool copyResult = false, TCustomDeepCopier customCopier = {});
- [[nodiscard]]
+ [[nodiscard]]
TExprNode::TPtr SwapWithHead(const TExprNode& node);
TExprNode::TPtr ReplaceNode(TExprNode::TPtr&& start, const TExprNode& src, TExprNode::TPtr dst);
TExprNode::TPtr ReplaceNodes(TExprNode::TPtr&& start, const TNodeOnNodeOwnedMap& replaces);
@@ -2432,9 +2432,9 @@ struct TExprContext : private TNonCopyable {
template <typename T, typename... Args>
const T* MakeType(Args&&... args);
- template <typename T, typename... Args>
- const T* MakeConstraint(Args&&... args);
-
+ template <typename T, typename... Args>
+ const T* MakeConstraint(Args&&... args);
+
void AddError(const TIssue& error) {
ENSURE_NOT_FROZEN_CTX
IssueManager.RaiseIssue(error);
@@ -2450,11 +2450,11 @@ struct TExprContext : private TNonCopyable {
void Reset();
- template <class TConstraint>
- bool IsConstraintEnabled() const {
- return DisabledConstraints.find(TConstraint::Name()) == DisabledConstraints.end();
- }
-
+ template <class TConstraint>
+ bool IsConstraintEnabled() const {
+ return DisabledConstraints.find(TConstraint::Name()) == DisabledConstraints.end();
+ }
+
private:
using TPositionHandleEqualPred = std::function<bool(TPositionHandle, TPositionHandle)>;
using TPositionHandleHasher = std::function<size_t(TPositionHandle)>;
@@ -2466,24 +2466,24 @@ private:
std::deque<TPosition> Positions;
};
-template <typename T, typename... Args>
-inline const T* TExprContext::MakeConstraint(Args&&... args) {
- ENSURE_NOT_FROZEN_CTX
- if (!IsConstraintEnabled<T>()) {
- return nullptr;
- }
-
- T sample(*this, std::forward<Args>(args)...);
- const auto it = ConstraintSet.find(&sample);
- if (ConstraintSet.cend() != it) {
- return static_cast<const T*>(*it);
- }
-
- ConstraintNodes.emplace(new T(std::move(sample)));
- const auto ins = ConstraintSet.emplace(ConstraintNodes.top().get());
- return static_cast<const T*>(*ins.first);
-}
-
+template <typename T, typename... Args>
+inline const T* TExprContext::MakeConstraint(Args&&... args) {
+ ENSURE_NOT_FROZEN_CTX
+ if (!IsConstraintEnabled<T>()) {
+ return nullptr;
+ }
+
+ T sample(*this, std::forward<Args>(args)...);
+ const auto it = ConstraintSet.find(&sample);
+ if (ConstraintSet.cend() != it) {
+ return static_cast<const T*>(*it);
+ }
+
+ ConstraintNodes.emplace(new T(std::move(sample)));
+ const auto ins = ConstraintSet.emplace(ConstraintNodes.top().get());
+ return static_cast<const T*>(*ins.first);
+}
+
#undef ENSURE_NOT_DELETED
#undef ENSURE_NOT_FROZEN
#undef ENSURE_NOT_FROZEN_CTX
diff --git a/ydb/library/yql/ast/yql_expr_builder.cpp b/ydb/library/yql/ast/yql_expr_builder.cpp
index 4b22308b99..ccb85512de 100644
--- a/ydb/library/yql/ast/yql_expr_builder.cpp
+++ b/ydb/library/yql/ast/yql_expr_builder.cpp
@@ -30,9 +30,9 @@ TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprNodeBuilder* parent
, Pos(pos)
, CurrentNode(nullptr)
{
- if (Parent) {
- ExtArgsFunc = Parent->ExtArgsFunc;
- }
+ if (Parent) {
+ ExtArgsFunc = Parent->ExtArgsFunc;
+ }
}
TExprNodeBuilder::TExprNodeBuilder(TPositionHandle pos, TExprNodeReplaceBuilder* parentReplacer)
@@ -82,7 +82,7 @@ TExprNodeBuilder& TExprNodeBuilder::Atom(ui32 index, TPositionHandle pos, const
Y_ENSURE(Container->ChildrenSize() == index,
"Container position mismatch, expected: " << Container->ChildrenSize() <<
", actual: " << index);
- auto child = Ctx.NewAtom(pos, content, flags);
+ auto child = Ctx.NewAtom(pos, content, flags);
Container->Children_.push_back(child);
return *this;
}
@@ -90,16 +90,16 @@ TExprNodeBuilder& TExprNodeBuilder::Atom(ui32 index, TPositionHandle pos, const
TExprNodeBuilder& TExprNodeBuilder::Atom(TPositionHandle pos, const TStringBuf& content, ui32 flags) {
Y_ENSURE(!Container || Container->IsLambda(), "No container expected");
Y_ENSURE(!CurrentNode, "Node is already build");
- CurrentNode = Ctx.NewAtom(pos, content, flags);
+ CurrentNode = Ctx.NewAtom(pos, content, flags);
return *this;
}
-TExprNodeBuilder& TExprNodeBuilder::Atom(ui32 index, const TStringBuf& content, ui32 flags) {
- return Atom(index, Pos, content, flags);
+TExprNodeBuilder& TExprNodeBuilder::Atom(ui32 index, const TStringBuf& content, ui32 flags) {
+ return Atom(index, Pos, content, flags);
}
-TExprNodeBuilder& TExprNodeBuilder::Atom(const TStringBuf& content, ui32 flags) {
- return Atom(Pos, content, flags);
+TExprNodeBuilder& TExprNodeBuilder::Atom(const TStringBuf& content, ui32 flags) {
+ return Atom(Pos, content, flags);
}
TExprNodeBuilder TExprNodeBuilder::List(ui32 index, TPositionHandle pos) {
@@ -295,14 +295,14 @@ TExprNodeBuilder& TExprNodeBuilder::Arg(const TStringBuf& name, ui32 toIndex) {
return Arg(TString(name) += ToString(toIndex));
}
-TExprNodeBuilder& TExprNodeBuilder::Arg(const TExprNodePtr& arg) {
- Y_ENSURE(arg->Type() == TExprNode::Argument, "Argument expected");
- Y_ENSURE(!Container || Container->Type() == TExprNode::Lambda, "No container expected");
- Y_ENSURE(!CurrentNode, "Node is already build");
- CurrentNode = arg;
- return *this;
-}
-
+TExprNodeBuilder& TExprNodeBuilder::Arg(const TExprNodePtr& arg) {
+ Y_ENSURE(arg->Type() == TExprNode::Argument, "Argument expected");
+ Y_ENSURE(!Container || Container->Type() == TExprNode::Lambda, "No container expected");
+ Y_ENSURE(!CurrentNode, "Node is already build");
+ CurrentNode = arg;
+ return *this;
+}
+
TExprNodeBuilder& TExprNodeBuilder::Args(ui32 index, const TStringBuf& name, ui32 toIndex) {
Y_ENSURE(!name.empty(), "Empty parameter name is not allowed");
for (auto i = 0U; index < toIndex; ++i)
diff --git a/ydb/library/yql/ast/yql_expr_builder.h b/ydb/library/yql/ast/yql_expr_builder.h
index 18c63f2eea..86151a6aaf 100644
--- a/ydb/library/yql/ast/yql_expr_builder.h
+++ b/ydb/library/yql/ast/yql_expr_builder.h
@@ -1,6 +1,6 @@
#pragma once
-#include "yql_ast.h"
+#include "yql_ast.h"
#include "yql_errors.h"
#include "yql_pos_handle.h"
@@ -30,8 +30,8 @@ public:
// non-indexed version must be used (at root or as lambda body)
TExprNodeBuilder& Atom(ui32 index, TPositionHandle pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent);
TExprNodeBuilder& Atom(TPositionHandle pos, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent);
- TExprNodeBuilder& Atom(ui32 index, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent);
- TExprNodeBuilder& Atom(const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent);
+ TExprNodeBuilder& Atom(ui32 index, const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent);
+ TExprNodeBuilder& Atom(const TStringBuf& content, ui32 flags = TNodeFlags::ArbitraryContent);
TExprNodeBuilder List(ui32 index, TPositionHandle pos);
TExprNodeBuilder List(TPositionHandle pos);
@@ -68,7 +68,7 @@ public:
TExprNodeBuilder& Arg(const TStringBuf& name);
TExprNodeBuilder& Arg(ui32 index, const TStringBuf& name, ui32 toIndex);
TExprNodeBuilder& Arg(const TStringBuf& name, ui32 toIndex);
- TExprNodeBuilder& Arg(const TExprNodePtr& arg);
+ TExprNodeBuilder& Arg(const TExprNodePtr& arg);
TExprNodeBuilder& Args(ui32 index, const TStringBuf& name, ui32 toIndex);
TExprNodeBuilder& Args(const TStringBuf& name, ui32 toIndex);
diff --git a/ydb/library/yql/ast/yql_expr_ut.cpp b/ydb/library/yql/ast/yql_expr_ut.cpp
index 06550f2507..a8e9635f08 100644
--- a/ydb/library/yql/ast/yql_expr_ut.cpp
+++ b/ydb/library/yql/ast/yql_expr_ut.cpp
@@ -15,7 +15,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) {
}
static void CompileExprWithCheck(TAstNode& root, TExprNode::TPtr& exprRoot, TExprContext& exprCtx, ui32 typeAnnotationIndex = Max<ui32>()) {
- const bool success = CompileExpr(root, exprRoot, exprCtx, nullptr, typeAnnotationIndex != Max<ui32>(), typeAnnotationIndex);
+ const bool success = CompileExpr(root, exprRoot, exprCtx, nullptr, typeAnnotationIndex != Max<ui32>(), typeAnnotationIndex);
exprCtx.IssueManager.GetIssues().PrintTo(Cout);
UNIT_ASSERT(success);
@@ -33,7 +33,7 @@ Y_UNIT_TEST_SUITE(TCompileYqlExpr) {
TAstParseResult astRes = ParseAstWithCheck(program);
TExprContext exprCtx;
TExprNode::TPtr exprRoot;
- bool result = CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr);
+ bool result = CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr);
exprCtx.IssueManager.GetIssues().PrintTo(Cout);
return result;
}
@@ -313,8 +313,8 @@ Y_UNIT_TEST_SUITE(TCompareExprTrees) {
TExprContext ctxOne, ctxTwo;
TExprNode::TPtr rootOne, rootTwo;
- UNIT_ASSERT(CompileExpr(*progOne.Root, rootOne, ctxOne, nullptr));
- UNIT_ASSERT(CompileExpr(*progTwo.Root, rootTwo, ctxTwo, nullptr));
+ UNIT_ASSERT(CompileExpr(*progOne.Root, rootOne, ctxOne, nullptr));
+ UNIT_ASSERT(CompileExpr(*progTwo.Root, rootTwo, ctxTwo, nullptr));
const TExprNode* diffOne = rootOne.Get();
const TExprNode* diffTwo = rootTwo.Get();
@@ -893,7 +893,7 @@ Y_UNIT_TEST_SUITE(TConvertToAst) {
UNIT_ASSERT(astRes.IsOk());
TExprContext exprCtx;
TExprNode::TPtr exprRoot;
- UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr));
+ UNIT_ASSERT(CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr));
UNIT_ASSERT(exprRoot);
const auto convRes = ConvertToAst(*exprRoot, exprCtx, 0, true);
@@ -901,7 +901,7 @@ Y_UNIT_TEST_SUITE(TConvertToAst) {
TExprContext exprCtx2;
TExprNode::TPtr exprRoot2;
- auto compileOk = CompileExpr(*convRes.Root, exprRoot2, exprCtx2, nullptr);
+ auto compileOk = CompileExpr(*convRes.Root, exprRoot2, exprCtx2, nullptr);
exprCtx2.IssueManager.GetIssues().PrintTo(Cout);
UNIT_ASSERT(compileOk);
UNIT_ASSERT(exprRoot2);
diff --git a/ydb/library/yql/core/common_opt/ya.make b/ydb/library/yql/core/common_opt/ya.make
index 518c91c26a..a503e6b5ae 100644
--- a/ydb/library/yql/core/common_opt/ya.make
+++ b/ydb/library/yql/core/common_opt/ya.make
@@ -1,15 +1,15 @@
LIBRARY()
-OWNER(
+OWNER(
vvvv
- g:yql
+ g:yql
g:yql_ydb_core
-)
+)
SRCS(
yql_co.h
- yql_co_extr_members.cpp
- yql_co_finalizers.cpp
+ yql_co_extr_members.cpp
+ yql_co_finalizers.cpp
yql_co_flow1.cpp
yql_co_flow2.cpp
yql_co_last.cpp
diff --git a/ydb/library/yql/core/common_opt/yql_co.h b/ydb/library/yql/core/common_opt/yql_co.h
index b8f2719037..701786b565 100644
--- a/ydb/library/yql/core/common_opt/yql_co.h
+++ b/ydb/library/yql/core/common_opt/yql_co.h
@@ -8,7 +8,7 @@
namespace NYql {
struct TOptimizeContext {
- TTypeAnnotationContext* Types = nullptr;
+ TTypeAnnotationContext* Types = nullptr;
TParentsMap* ParentsMap = nullptr;
const TExprNode* GetParentIfSingle(const TExprNode& node) const {
@@ -33,7 +33,7 @@ struct TOptimizeContext {
YQL_ENSURE(usageCount == 1);
return *parents.cbegin();
}
-
+
bool IsSingleUsage(const TExprNode& node) const {
return bool(GetParentIfSingle(node));
}
@@ -47,23 +47,23 @@ struct TOptimizeContext {
return ParentsMap->contains(&node);
}
- bool IsPersistentNode(const TExprNode& node) const {
- if (Types) {
- for (auto& source: Types->DataSources) {
- if (source->IsPersistent(node)) {
- return true;
- }
- }
-
- for (auto& sink: Types->DataSinks) {
- if (sink->IsPersistent(node)) {
- return true;
- }
- }
- }
-
- return false;
- }
+ bool IsPersistentNode(const TExprNode& node) const {
+ if (Types) {
+ for (auto& source: Types->DataSources) {
+ if (source->IsPersistent(node)) {
+ return true;
+ }
+ }
+
+ for (auto& sink: Types->DataSinks) {
+ if (sink->IsPersistent(node)) {
+ return true;
+ }
+ }
+ }
+
+ return false;
+ }
bool IsPersistentNode(const NNodes::TExprBase& node) const {
return IsPersistentNode(node.Ref());
@@ -72,7 +72,7 @@ struct TOptimizeContext {
using TCallableOptimizerExt = std::function<TExprNode::TPtr (const TExprNode::TPtr&, TExprContext&, TOptimizeContext&)>;
using TCallableOptimizerMap = std::unordered_map<std::string_view, TCallableOptimizerExt>;
-using TFinalizingOptimizerExt = std::function<void (const TExprNode::TPtr&, TNodeOnNodeOwnedMap&, TExprContext&, TOptimizeContext&)>;
+using TFinalizingOptimizerExt = std::function<void (const TExprNode::TPtr&, TNodeOnNodeOwnedMap&, TExprContext&, TOptimizeContext&)>;
using TFinalizingOptimizerMap = std::unordered_map<std::string_view, TFinalizingOptimizerExt>;
struct TCoCallableRules {
@@ -93,8 +93,8 @@ struct TCoCallableRules {
// rules that make a flow fork - Join pushdown if Join has multiple usage
TCallableOptimizerMap FlowCallables[FLOW_STEPS];
- TFinalizingOptimizerMap Finalizers;
-
+ TFinalizingOptimizerMap Finalizers;
+
// rules to be applied before execution
TCallableOptimizerMap FinalCallables;
@@ -106,7 +106,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map);
void RegisterCoSimpleCallables2(TCallableOptimizerMap& map);
void RegisterCoFlowCallables1(TCallableOptimizerMap& map);
void RegisterCoFlowCallables2(TCallableOptimizerMap& map);
-void RegisterCoFinalizers(TFinalizingOptimizerMap& map);
+void RegisterCoFinalizers(TFinalizingOptimizerMap& map);
void RegisterCoFinalCallables(TCallableOptimizerMap& map);
}
diff --git a/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp b/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp
index 215a1e9f69..3f12dd971b 100644
--- a/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_extr_members.cpp
@@ -1,441 +1,441 @@
-#include "yql_co_extr_members.h"
-
+#include "yql_co_extr_members.h"
+
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
#include <ydb/library/yql/core/yql_join.h>
#include <ydb/library/yql/core/yql_opt_utils.h>
#include <ydb/library/yql/core/yql_opt_window.h>
-
+
#include <ydb/library/yql/utils/log/log.h>
-
-namespace NYql {
-
-using namespace NNodes;
-
-TExprNode::TPtr ApplyExtractMembersToTake(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
- TCoTake take(node);
+
+namespace NYql {
+
+using namespace NNodes;
+
+TExprNode::TPtr ApplyExtractMembersToTake(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoTake take(node);
YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
- return Build<TCoTake>(ctx, node->Pos())
- .Input<TCoExtractMembers>()
- .Input(take.Input())
- .Members(members)
- .Build()
- .Count(take.Count())
- .Done().Ptr();
-}
-
-TExprNode::TPtr ApplyExtractMembersToSkip(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
- TCoSkip skip(node);
- YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
- return Build<TCoSkip>(ctx, node->Pos())
- .Input<TCoExtractMembers>()
- .Input(skip.Input())
- .Members(members)
- .Build()
- .Count(skip.Count())
- .Done().Ptr();
-}
-
-TExprNode::TPtr ApplyExtractMembersToExtend(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+ return Build<TCoTake>(ctx, node->Pos())
+ .Input<TCoExtractMembers>()
+ .Input(take.Input())
+ .Members(members)
+ .Build()
+ .Count(take.Count())
+ .Done().Ptr();
+}
+
+TExprNode::TPtr ApplyExtractMembersToSkip(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoSkip skip(node);
YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
- TExprNode::TListType inputs;
- for (auto& child: node->Children()) {
- inputs.emplace_back(ctx.Builder(child->Pos())
- .Callable(TCoExtractMembers::CallableName())
- .Add(0, child)
- .Add(1, members)
- .Seal()
- .Build());
- }
-
- return ctx.NewCallable(node->Pos(), node->Content(), std::move(inputs));
-}
-
-TExprNode::TPtr ApplyExtractMembersToSkipNullMembers(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
- TCoSkipNullMembers skipNullMembers(node);
+ return Build<TCoSkip>(ctx, node->Pos())
+ .Input<TCoExtractMembers>()
+ .Input(skip.Input())
+ .Members(members)
+ .Build()
+ .Count(skip.Count())
+ .Done().Ptr();
+}
+
+TExprNode::TPtr ApplyExtractMembersToExtend(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+ YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
+ TExprNode::TListType inputs;
+ for (auto& child: node->Children()) {
+ inputs.emplace_back(ctx.Builder(child->Pos())
+ .Callable(TCoExtractMembers::CallableName())
+ .Add(0, child)
+ .Add(1, members)
+ .Seal()
+ .Build());
+ }
+
+ return ctx.NewCallable(node->Pos(), node->Content(), std::move(inputs));
+}
+
+TExprNode::TPtr ApplyExtractMembersToSkipNullMembers(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoSkipNullMembers skipNullMembers(node);
const auto& filtered = skipNullMembers.Members();
if (!filtered) {
return {};
}
- TExprNode::TListType filteredMembers;
+ TExprNode::TListType filteredMembers;
for (const auto& x : filtered.Cast()) {
- auto member = x.Value();
- bool hasMember = false;
- for (const auto& y : members->ChildrenList()) {
- if (member == y->Content()) {
- hasMember = true;
- break;
- }
- }
-
- if (hasMember) {
- filteredMembers.push_back(x.Ptr());
- }
- }
-
+ auto member = x.Value();
+ bool hasMember = false;
+ for (const auto& y : members->ChildrenList()) {
+ if (member == y->Content()) {
+ hasMember = true;
+ break;
+ }
+ }
+
+ if (hasMember) {
+ filteredMembers.push_back(x.Ptr());
+ }
+ }
+
YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
- return Build<TCoSkipNullMembers>(ctx, skipNullMembers.Pos())
- .Input<TCoExtractMembers>()
- .Input(skipNullMembers.Input())
- .Members(members)
- .Build()
- .Members(ctx.NewList(skipNullMembers.Pos(), std::move(filteredMembers)))
- .Done().Ptr();
-}
-
-TExprNode::TPtr ApplyExtractMembersToFilterNullMembers(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
- TCoFilterNullMembers filterNullMembers(node);
- if (!filterNullMembers.Input().Maybe<TCoAssumeAllMembersNullableAtOnce>()) {
- return {};
- }
- auto input = filterNullMembers.Input().Cast<TCoAssumeAllMembersNullableAtOnce>().Input();
-
- const auto originalStructType = GetSeqItemType(filterNullMembers.Input().Ref().GetTypeAnn())->Cast<TStructExprType>();
-
- TExprNode::TPtr extendedMembers;
- TMaybeNode<TCoAtomList> filteredMembers;
- if (const auto& filtered = filterNullMembers.Members()) {
- TExprNode::TListType updatedMembers;
- for (const auto& x : filtered.Cast()) {
- auto member = x.Value();
- bool hasMember = false;
- for (const auto& y : members->ChildrenList()) {
- if (member == y->Content()) {
- hasMember = true;
- break;
- }
- }
-
- if (hasMember) {
- updatedMembers.push_back(x.Ptr());
- }
- }
- if ((members->ChildrenList().size() + updatedMembers.empty()) == originalStructType->GetSize()) {
- return {};
- }
- if (updatedMembers.empty()) {
- // Keep at least one optional field in input
- const auto extra = filtered.Cast().Item(0).Ptr();
- updatedMembers.push_back(extra);
- auto list = members->ChildrenList();
- list.push_back(extra);
- extendedMembers = ctx.NewList(members->Pos(), std::move(list));
- }
- filteredMembers = TCoAtomList(ctx.NewList(filtered.Cast().Pos(), std::move(updatedMembers)));
- } else {
-
- bool hasOptional = false;
- for (const auto& y : members->ChildrenList()) {
- if (auto type = originalStructType->FindItemType(y->Content()); type->GetKind() == ETypeAnnotationKind::Optional) {
- hasOptional = true;
- break;
- }
- }
-
- if ((members->ChildrenList().size() + !hasOptional) == originalStructType->GetSize()) {
- return {};
- }
-
- if (!hasOptional) {
- // Keep at least one optional field in input (use first any optional field)
- for (const auto& x : originalStructType->GetItems()) {
- if (x->GetItemType()->GetKind() == ETypeAnnotationKind::Optional) {
- auto list = members->ChildrenList();
- list.push_back(ctx.NewAtom(members->Pos(), x->GetName()));
- extendedMembers = ctx.NewList(members->Pos(), std::move(list));
- break;
- }
- }
- YQL_ENSURE(extendedMembers);
- }
- }
-
- YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
-
- if (extendedMembers) {
- return Build<TCoExtractMembers>(ctx, filterNullMembers.Pos())
- .Input<TCoFilterNullMembers>()
- .Input<TCoExtractMembers>()
- .Input(input)
- .Members(extendedMembers)
- .Build()
- .Members(filteredMembers)
- .Build()
- .Members(members)
- .Done().Ptr();
- }
-
- return Build<TCoFilterNullMembers>(ctx, filterNullMembers.Pos())
- .Input<TCoExtractMembers>()
- .Input(input)
- .Members(members)
- .Build()
- .Members(filteredMembers)
- .Done().Ptr();
-}
-
-TExprNode::TPtr ApplyExtractMembersToSort(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix) {
- TCoSortBase sort(node);
- TSet<TStringBuf> extractFields;
- for (const auto& x : members->ChildrenList()) {
- extractFields.emplace(x->Content());
- }
- TSet<TStringBuf> sortKeys;
+ return Build<TCoSkipNullMembers>(ctx, skipNullMembers.Pos())
+ .Input<TCoExtractMembers>()
+ .Input(skipNullMembers.Input())
+ .Members(members)
+ .Build()
+ .Members(ctx.NewList(skipNullMembers.Pos(), std::move(filteredMembers)))
+ .Done().Ptr();
+}
+
+TExprNode::TPtr ApplyExtractMembersToFilterNullMembers(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoFilterNullMembers filterNullMembers(node);
+ if (!filterNullMembers.Input().Maybe<TCoAssumeAllMembersNullableAtOnce>()) {
+ return {};
+ }
+ auto input = filterNullMembers.Input().Cast<TCoAssumeAllMembersNullableAtOnce>().Input();
+
+ const auto originalStructType = GetSeqItemType(filterNullMembers.Input().Ref().GetTypeAnn())->Cast<TStructExprType>();
+
+ TExprNode::TPtr extendedMembers;
+ TMaybeNode<TCoAtomList> filteredMembers;
+ if (const auto& filtered = filterNullMembers.Members()) {
+ TExprNode::TListType updatedMembers;
+ for (const auto& x : filtered.Cast()) {
+ auto member = x.Value();
+ bool hasMember = false;
+ for (const auto& y : members->ChildrenList()) {
+ if (member == y->Content()) {
+ hasMember = true;
+ break;
+ }
+ }
+
+ if (hasMember) {
+ updatedMembers.push_back(x.Ptr());
+ }
+ }
+ if ((members->ChildrenList().size() + updatedMembers.empty()) == originalStructType->GetSize()) {
+ return {};
+ }
+ if (updatedMembers.empty()) {
+ // Keep at least one optional field in input
+ const auto extra = filtered.Cast().Item(0).Ptr();
+ updatedMembers.push_back(extra);
+ auto list = members->ChildrenList();
+ list.push_back(extra);
+ extendedMembers = ctx.NewList(members->Pos(), std::move(list));
+ }
+ filteredMembers = TCoAtomList(ctx.NewList(filtered.Cast().Pos(), std::move(updatedMembers)));
+ } else {
+
+ bool hasOptional = false;
+ for (const auto& y : members->ChildrenList()) {
+ if (auto type = originalStructType->FindItemType(y->Content()); type->GetKind() == ETypeAnnotationKind::Optional) {
+ hasOptional = true;
+ break;
+ }
+ }
+
+ if ((members->ChildrenList().size() + !hasOptional) == originalStructType->GetSize()) {
+ return {};
+ }
+
+ if (!hasOptional) {
+ // Keep at least one optional field in input (use first any optional field)
+ for (const auto& x : originalStructType->GetItems()) {
+ if (x->GetItemType()->GetKind() == ETypeAnnotationKind::Optional) {
+ auto list = members->ChildrenList();
+ list.push_back(ctx.NewAtom(members->Pos(), x->GetName()));
+ extendedMembers = ctx.NewList(members->Pos(), std::move(list));
+ break;
+ }
+ }
+ YQL_ENSURE(extendedMembers);
+ }
+ }
+
+ YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
+
+ if (extendedMembers) {
+ return Build<TCoExtractMembers>(ctx, filterNullMembers.Pos())
+ .Input<TCoFilterNullMembers>()
+ .Input<TCoExtractMembers>()
+ .Input(input)
+ .Members(extendedMembers)
+ .Build()
+ .Members(filteredMembers)
+ .Build()
+ .Members(members)
+ .Done().Ptr();
+ }
+
+ return Build<TCoFilterNullMembers>(ctx, filterNullMembers.Pos())
+ .Input<TCoExtractMembers>()
+ .Input(input)
+ .Members(members)
+ .Build()
+ .Members(filteredMembers)
+ .Done().Ptr();
+}
+
+TExprNode::TPtr ApplyExtractMembersToSort(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoSortBase sort(node);
+ TSet<TStringBuf> extractFields;
+ for (const auto& x : members->ChildrenList()) {
+ extractFields.emplace(x->Content());
+ }
+ TSet<TStringBuf> sortKeys;
bool fieldSubset = HaveFieldsSubset(sort.KeySelectorLambda().Body().Ptr(), sort.KeySelectorLambda().Args().Arg(0).Ref(), sortKeys, parentsMap);
- bool allExist = true;
- if (!sortKeys.empty()) {
- for (const auto& key : sortKeys) {
+ bool allExist = true;
+ if (!sortKeys.empty()) {
+ for (const auto& key : sortKeys) {
auto ret = extractFields.emplace(key);
if (ret.second) {
- allExist = false;
- }
- }
- }
+ allExist = false;
+ }
+ }
+ }
if (allExist && sortKeys.size() == extractFields.size()) {
YQL_CLOG(DEBUG, Core) << "Force `fieldSubset` for ExtractMembers over " << node->Content();
fieldSubset = true;
}
- if (fieldSubset && allExist) {
- YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
- return ctx.Builder(sort.Pos())
- .Callable(node->Content())
- .Callable(0, TCoExtractMembers::CallableName())
- .Add(0, sort.Input().Ptr())
- .Add(1, members)
- .Seal()
- .Add(1, sort.SortDirections().Ptr())
- .Add(2, ctx.DeepCopyLambda(sort.KeySelectorLambda().Ref()))
- .Seal()
- .Build();
- }
- else if (fieldSubset) {
+ if (fieldSubset && allExist) {
+ YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
+ return ctx.Builder(sort.Pos())
+ .Callable(node->Content())
+ .Callable(0, TCoExtractMembers::CallableName())
+ .Add(0, sort.Input().Ptr())
+ .Add(1, members)
+ .Seal()
+ .Add(1, sort.SortDirections().Ptr())
+ .Add(2, ctx.DeepCopyLambda(sort.KeySelectorLambda().Ref()))
+ .Seal()
+ .Build();
+ }
+ else if (fieldSubset) {
const auto structType = GetSeqItemType(sort.Ref().GetTypeAnn())->Cast<TStructExprType>();
- if (structType->GetSize() <= extractFields.size()) {
- return {};
- }
- YQL_CLOG(DEBUG, Core) << "Inject ExtractMembers into " << node->Content() << logSuffix;
- TExprNode::TListType totalExtracted;
- for (const auto& field : extractFields) {
- totalExtracted.emplace_back(ctx.NewAtom(members->Pos(), field));
- }
-
- return ctx.Builder(sort.Pos())
- .Callable(TCoExtractMembers::CallableName())
- .Callable(0, node->Content())
- .Callable(0, TCoExtractMembers::CallableName())
- .Add(0, sort.Input().Ptr())
- .Add(1, ctx.NewList(members->Pos(), std::move(totalExtracted)))
- .Seal()
- .Add(1, sort.SortDirections().Ptr())
- .Add(2, ctx.DeepCopyLambda(sort.KeySelectorLambda().Ref()))
- .Seal()
- .Add(1, members)
- .Seal()
- .Build();
- }
- return {};
-}
-
-TExprNode::TPtr ApplyExtractMembersToAssumeUnique(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
- TCoAssumeUnique assumeUnique(node);
- TSet<TStringBuf> extractFields;
- for (const auto& x : members->ChildrenList()) {
- extractFields.emplace(x->Content());
- }
- const bool allExist = AllOf(assumeUnique.UniqueBy(), [&extractFields] (const TCoAtom& u) { return extractFields.contains(u.Value()); });
- if (allExist) {
- YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
- return ctx.Builder(assumeUnique.Pos())
- .Callable(node->Content())
- .Callable(0, TCoExtractMembers::CallableName())
- .Add(0, assumeUnique.Input().Ptr())
- .Add(1, members)
- .Seal()
- .Add(1, assumeUnique.UniqueBy().Ptr())
- .Seal()
- .Build();
- }
- YQL_CLOG(DEBUG, Core) << "Drop " << node->Content() << " after ExtractMembers" << logSuffix;
- return ctx.Builder(assumeUnique.Pos())
- .Callable(TCoExtractMembers::CallableName())
- .Add(0, assumeUnique.Input().Ptr())
- .Add(1, members)
- .Seal()
- .Build();
-}
-
-TExprNode::TPtr ApplyExtractMembersToTop(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix) {
- TCoTopBase top(node);
- TSet<TStringBuf> extractFields;
- for (const auto& x : members->ChildrenList()) {
- extractFields.emplace(x->Content());
- }
- TSet<TStringBuf> sortKeys;
- const bool fieldSubset = HaveFieldsSubset(top.KeySelectorLambda().Body().Ptr(), top.KeySelectorLambda().Args().Arg(0).Ref(), sortKeys, parentsMap);
- bool allExist = true;
- if (!sortKeys.empty()) {
- for (const auto& key : sortKeys) {
- if (!extractFields.contains(key)) {
- allExist = false;
- extractFields.emplace(key);
- }
- }
- }
- if (fieldSubset && allExist) {
- YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
- auto children = node->ChildrenList();
- children[TCoTopBase::idx_Input] = Build<TCoExtractMembers>(ctx, top.Pos())
- .Input(top.Input())
- .Members(members)
- .Done().Ptr();
- children[TCoTopBase::idx_KeySelectorLambda] = ctx.DeepCopyLambda(top.KeySelectorLambda().Ref());
- return ctx.ChangeChildren(*node, std::move(children));
- }
- else if (fieldSubset) {
+ if (structType->GetSize() <= extractFields.size()) {
+ return {};
+ }
+ YQL_CLOG(DEBUG, Core) << "Inject ExtractMembers into " << node->Content() << logSuffix;
+ TExprNode::TListType totalExtracted;
+ for (const auto& field : extractFields) {
+ totalExtracted.emplace_back(ctx.NewAtom(members->Pos(), field));
+ }
+
+ return ctx.Builder(sort.Pos())
+ .Callable(TCoExtractMembers::CallableName())
+ .Callable(0, node->Content())
+ .Callable(0, TCoExtractMembers::CallableName())
+ .Add(0, sort.Input().Ptr())
+ .Add(1, ctx.NewList(members->Pos(), std::move(totalExtracted)))
+ .Seal()
+ .Add(1, sort.SortDirections().Ptr())
+ .Add(2, ctx.DeepCopyLambda(sort.KeySelectorLambda().Ref()))
+ .Seal()
+ .Add(1, members)
+ .Seal()
+ .Build();
+ }
+ return {};
+}
+
+TExprNode::TPtr ApplyExtractMembersToAssumeUnique(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoAssumeUnique assumeUnique(node);
+ TSet<TStringBuf> extractFields;
+ for (const auto& x : members->ChildrenList()) {
+ extractFields.emplace(x->Content());
+ }
+ const bool allExist = AllOf(assumeUnique.UniqueBy(), [&extractFields] (const TCoAtom& u) { return extractFields.contains(u.Value()); });
+ if (allExist) {
+ YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
+ return ctx.Builder(assumeUnique.Pos())
+ .Callable(node->Content())
+ .Callable(0, TCoExtractMembers::CallableName())
+ .Add(0, assumeUnique.Input().Ptr())
+ .Add(1, members)
+ .Seal()
+ .Add(1, assumeUnique.UniqueBy().Ptr())
+ .Seal()
+ .Build();
+ }
+ YQL_CLOG(DEBUG, Core) << "Drop " << node->Content() << " after ExtractMembers" << logSuffix;
+ return ctx.Builder(assumeUnique.Pos())
+ .Callable(TCoExtractMembers::CallableName())
+ .Add(0, assumeUnique.Input().Ptr())
+ .Add(1, members)
+ .Seal()
+ .Build();
+}
+
+TExprNode::TPtr ApplyExtractMembersToTop(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoTopBase top(node);
+ TSet<TStringBuf> extractFields;
+ for (const auto& x : members->ChildrenList()) {
+ extractFields.emplace(x->Content());
+ }
+ TSet<TStringBuf> sortKeys;
+ const bool fieldSubset = HaveFieldsSubset(top.KeySelectorLambda().Body().Ptr(), top.KeySelectorLambda().Args().Arg(0).Ref(), sortKeys, parentsMap);
+ bool allExist = true;
+ if (!sortKeys.empty()) {
+ for (const auto& key : sortKeys) {
+ if (!extractFields.contains(key)) {
+ allExist = false;
+ extractFields.emplace(key);
+ }
+ }
+ }
+ if (fieldSubset && allExist) {
+ YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
+ auto children = node->ChildrenList();
+ children[TCoTopBase::idx_Input] = Build<TCoExtractMembers>(ctx, top.Pos())
+ .Input(top.Input())
+ .Members(members)
+ .Done().Ptr();
+ children[TCoTopBase::idx_KeySelectorLambda] = ctx.DeepCopyLambda(top.KeySelectorLambda().Ref());
+ return ctx.ChangeChildren(*node, std::move(children));
+ }
+ else if (fieldSubset) {
const auto structType = GetSeqItemType(top.Ref().GetTypeAnn())->Cast<TStructExprType>();
- if (structType->GetSize() <= extractFields.size()) {
- return {};
- }
- YQL_CLOG(DEBUG, Core) << "Inject ExtractMembers into " << node->Content() << logSuffix;
- TExprNode::TListType totalExtracted;
- for (const auto& field : extractFields) {
- totalExtracted.emplace_back(ctx.NewAtom(members->Pos(), field));
- }
-
- auto children = node->ChildrenList();
- children[TCoTopBase::idx_Input] = Build<TCoExtractMembers>(ctx, top.Pos())
- .Input(top.Input())
- .Members(ctx.NewList(members->Pos(), std::move(totalExtracted)))
- .Done().Ptr();
- children[TCoTopBase::idx_KeySelectorLambda] = ctx.DeepCopyLambda(top.KeySelectorLambda().Ref());
- auto updatedTop = ctx.ChangeChildren(*node, std::move(children));
-
- return Build<TCoExtractMembers>(ctx, top.Pos())
- .Input(updatedTop)
- .Members(members)
- .Done().Ptr();
- }
- return {};
-}
-
-TExprNode::TPtr ApplyExtractMembersToEquiJoin(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
- TCoEquiJoin join(node);
+ if (structType->GetSize() <= extractFields.size()) {
+ return {};
+ }
+ YQL_CLOG(DEBUG, Core) << "Inject ExtractMembers into " << node->Content() << logSuffix;
+ TExprNode::TListType totalExtracted;
+ for (const auto& field : extractFields) {
+ totalExtracted.emplace_back(ctx.NewAtom(members->Pos(), field));
+ }
+
+ auto children = node->ChildrenList();
+ children[TCoTopBase::idx_Input] = Build<TCoExtractMembers>(ctx, top.Pos())
+ .Input(top.Input())
+ .Members(ctx.NewList(members->Pos(), std::move(totalExtracted)))
+ .Done().Ptr();
+ children[TCoTopBase::idx_KeySelectorLambda] = ctx.DeepCopyLambda(top.KeySelectorLambda().Ref());
+ auto updatedTop = ctx.ChangeChildren(*node, std::move(children));
+
+ return Build<TCoExtractMembers>(ctx, top.Pos())
+ .Input(updatedTop)
+ .Members(members)
+ .Done().Ptr();
+ }
+ return {};
+}
+
+TExprNode::TPtr ApplyExtractMembersToEquiJoin(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoEquiJoin join(node);
const auto structType = GetSeqItemType(join.Ref().GetTypeAnn())->Cast<TStructExprType>();
- if (structType->GetSize() == 0) {
- return {};
- }
-
+ if (structType->GetSize() == 0) {
+ return {};
+ }
+
YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
- auto joinSettings = join.Arg(join.ArgCount() - 1).Ptr();
- auto renameMap = LoadJoinRenameMap(*joinSettings);
- joinSettings = RemoveSetting(*joinSettings, "rename", ctx);
- TSet<TStringBuf> usedFields;
- for (const auto& x : members->ChildrenList()) {
- usedFields.emplace(x->Content());
- }
-
- auto newRenameMap = UpdateUsedFieldsInRenameMap(renameMap, usedFields, structType);
- TExprNode::TListType joinSettingsNodes = joinSettings->ChildrenList();
- AppendEquiJoinRenameMap(join.Pos(), newRenameMap, joinSettingsNodes, ctx);
- joinSettings = ctx.ChangeChildren(*joinSettings, std::move(joinSettingsNodes));
- auto updatedEquiJoin = ctx.ShallowCopy(join.Ref());
- updatedEquiJoin->ChildRef(updatedEquiJoin->ChildrenSize() - 1) = joinSettings;
- return updatedEquiJoin;
-}
-
-TExprNode::TPtr ApplyExtractMembersToFlatMap(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
- TCoFlatMapBase flatmap(node);
- YQL_CLOG(DEBUG, Core) << "Apply ExtractMembers to " << node->Content() << logSuffix;
-
- auto body = flatmap.Lambda().Body();
- TMaybeNode<TExprBase> newBody;
-
- if (auto maybeConditional = body.Maybe<TCoConditionalValueBase>()) {
- auto conditional = maybeConditional.Cast();
- TMaybeNode<TExprBase> extracted;
-
- if (body.Maybe<TCoListIf>() || body.Maybe<TCoOptionalIf>()) {
- TVector<TExprBase> tuples;
- for (const auto& member : members->ChildrenList()) {
- auto tuple = Build<TCoNameValueTuple>(ctx, flatmap.Pos())
- .Name(member)
- .Value<TCoMember>()
- .Struct(conditional.Value())
- .Name(member)
- .Build()
- .Done();
-
- tuples.push_back(tuple);
- }
-
- extracted = Build<TCoAsStruct>(ctx, flatmap.Pos())
- .Add(tuples)
- .Done();
- } else {
- extracted = Build<TCoExtractMembers>(ctx, flatmap.Pos())
- .Input(conditional.Value())
- .Members(members)
- .Done();
- }
-
- newBody = ctx.ChangeChild(conditional.Ref(), TCoConditionalValueBase::idx_Value, extracted.Cast().Ptr());
- } else {
- newBody = Build<TCoExtractMembers>(ctx, flatmap.Pos())
- .Input(flatmap.Lambda().Body())
- .Members(members)
- .Done();
- }
-
- if (flatmap.Maybe<TCoOrderedFlatMap>()) {
- return Build<TCoOrderedFlatMap>(ctx, flatmap.Pos())
- .Input(flatmap.Input())
- .Lambda()
- .Args({"item"})
- .template Body<TExprApplier>()
- .Apply(newBody.Cast())
- .With(flatmap.Lambda().Args().Arg(0), "item")
- .Build()
- .Build()
- .Done()
- .Ptr();
- } else {
- return Build<TCoFlatMap>(ctx, flatmap.Pos())
- .Input(flatmap.Input())
- .Lambda()
- .Args({"item"})
- .template Body<TExprApplier>()
- .Apply(newBody.Cast())
- .With(flatmap.Lambda().Args().Arg(0), "item")
- .Build()
- .Build()
- .Done()
- .Ptr();
- }
-}
-
-TExprNode::TPtr ApplyExtractMembersToPartitionByKey(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
- TCoPartitionByKey part(node);
+ auto joinSettings = join.Arg(join.ArgCount() - 1).Ptr();
+ auto renameMap = LoadJoinRenameMap(*joinSettings);
+ joinSettings = RemoveSetting(*joinSettings, "rename", ctx);
+ TSet<TStringBuf> usedFields;
+ for (const auto& x : members->ChildrenList()) {
+ usedFields.emplace(x->Content());
+ }
+
+ auto newRenameMap = UpdateUsedFieldsInRenameMap(renameMap, usedFields, structType);
+ TExprNode::TListType joinSettingsNodes = joinSettings->ChildrenList();
+ AppendEquiJoinRenameMap(join.Pos(), newRenameMap, joinSettingsNodes, ctx);
+ joinSettings = ctx.ChangeChildren(*joinSettings, std::move(joinSettingsNodes));
+ auto updatedEquiJoin = ctx.ShallowCopy(join.Ref());
+ updatedEquiJoin->ChildRef(updatedEquiJoin->ChildrenSize() - 1) = joinSettings;
+ return updatedEquiJoin;
+}
+
+TExprNode::TPtr ApplyExtractMembersToFlatMap(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoFlatMapBase flatmap(node);
+ YQL_CLOG(DEBUG, Core) << "Apply ExtractMembers to " << node->Content() << logSuffix;
+
+ auto body = flatmap.Lambda().Body();
+ TMaybeNode<TExprBase> newBody;
+
+ if (auto maybeConditional = body.Maybe<TCoConditionalValueBase>()) {
+ auto conditional = maybeConditional.Cast();
+ TMaybeNode<TExprBase> extracted;
+
+ if (body.Maybe<TCoListIf>() || body.Maybe<TCoOptionalIf>()) {
+ TVector<TExprBase> tuples;
+ for (const auto& member : members->ChildrenList()) {
+ auto tuple = Build<TCoNameValueTuple>(ctx, flatmap.Pos())
+ .Name(member)
+ .Value<TCoMember>()
+ .Struct(conditional.Value())
+ .Name(member)
+ .Build()
+ .Done();
+
+ tuples.push_back(tuple);
+ }
+
+ extracted = Build<TCoAsStruct>(ctx, flatmap.Pos())
+ .Add(tuples)
+ .Done();
+ } else {
+ extracted = Build<TCoExtractMembers>(ctx, flatmap.Pos())
+ .Input(conditional.Value())
+ .Members(members)
+ .Done();
+ }
+
+ newBody = ctx.ChangeChild(conditional.Ref(), TCoConditionalValueBase::idx_Value, extracted.Cast().Ptr());
+ } else {
+ newBody = Build<TCoExtractMembers>(ctx, flatmap.Pos())
+ .Input(flatmap.Lambda().Body())
+ .Members(members)
+ .Done();
+ }
+
+ if (flatmap.Maybe<TCoOrderedFlatMap>()) {
+ return Build<TCoOrderedFlatMap>(ctx, flatmap.Pos())
+ .Input(flatmap.Input())
+ .Lambda()
+ .Args({"item"})
+ .template Body<TExprApplier>()
+ .Apply(newBody.Cast())
+ .With(flatmap.Lambda().Args().Arg(0), "item")
+ .Build()
+ .Build()
+ .Done()
+ .Ptr();
+ } else {
+ return Build<TCoFlatMap>(ctx, flatmap.Pos())
+ .Input(flatmap.Input())
+ .Lambda()
+ .Args({"item"})
+ .template Body<TExprApplier>()
+ .Apply(newBody.Cast())
+ .With(flatmap.Lambda().Args().Arg(0), "item")
+ .Build()
+ .Build()
+ .Done()
+ .Ptr();
+ }
+}
+
+TExprNode::TPtr ApplyExtractMembersToPartitionByKey(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoPartitionByKey part(node);
YQL_CLOG(DEBUG, Core) << "Apply ExtractMembers to " << node->Content() << logSuffix;
- auto newBody = Build<TCoExtractMembers>(ctx, part.Pos())
- .Input(part.ListHandlerLambda().Body())
- .Members(members)
- .Done();
-
- return Build<TCoPartitionByKey>(ctx, part.Pos())
- .Input(part.Input())
- .KeySelectorLambda(part.KeySelectorLambda())
- .ListHandlerLambda()
- .Args({"groups"})
- .Body<TExprApplier>()
- .Apply(newBody)
- .With(part.ListHandlerLambda().Args().Arg(0), "groups")
- .Build()
- .Build()
- .SortDirections(part.SortDirections())
- .SortKeySelectorLambda(part.SortKeySelectorLambda())
- .Done()
- .Ptr();
-}
-
+ auto newBody = Build<TCoExtractMembers>(ctx, part.Pos())
+ .Input(part.ListHandlerLambda().Body())
+ .Members(members)
+ .Done();
+
+ return Build<TCoPartitionByKey>(ctx, part.Pos())
+ .Input(part.Input())
+ .KeySelectorLambda(part.KeySelectorLambda())
+ .ListHandlerLambda()
+ .Args({"groups"})
+ .Body<TExprApplier>()
+ .Apply(newBody)
+ .With(part.ListHandlerLambda().Args().Arg(0), "groups")
+ .Build()
+ .Build()
+ .SortDirections(part.SortDirections())
+ .SortKeySelectorLambda(part.SortKeySelectorLambda())
+ .Done()
+ .Ptr();
+}
+
TExprNode::TPtr ApplyExtractMembersToChopper(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
const TCoChopper chopper(node);
YQL_CLOG(DEBUG, Core) << "Apply ExtractMembers to " << node->Content() << logSuffix;
@@ -512,28 +512,28 @@ TExprNode::TPtr ApplyExtractMembersToMapJoinCore(const TExprNode::TPtr& node, co
.Done().Ptr();
}
-TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
+TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
YQL_ENSURE(node->IsCallable({"CalcOverWindow", "CalcOverSessionWindow", "CalcOverWindowGroup"}));
auto input = node->ChildPtr(0);
- // window output = input fields + payload fields
- TSet<TStringBuf> outMembers;
- for (const auto& x : members->ChildrenList()) {
- outMembers.insert(x->Content());
- }
-
+ // window output = input fields + payload fields
+ TSet<TStringBuf> outMembers;
+ for (const auto& x : members->ChildrenList()) {
+ outMembers.insert(x->Content());
+ }
+
auto inputStructType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
auto outputStructType = node->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
- TSet<TStringBuf> toDrop;
- for (const auto& out : outputStructType->GetItems()) {
+ TSet<TStringBuf> toDrop;
+ for (const auto& out : outputStructType->GetItems()) {
if (!outMembers.contains(out->GetName())) {
- toDrop.insert(out->GetName());
- }
- }
-
+ toDrop.insert(out->GetName());
+ }
+ }
+
TSet<TStringBuf> usedFields;
- TSet<TStringBuf> payloadFields;
+ TSet<TStringBuf> payloadFields;
TExprNodeList newCalcs;
auto calcs = ExtractCalcsOverWindow(node, ctx);
bool dropped = false;
@@ -543,8 +543,8 @@ TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node,
// all partition keys will be used
for (const auto& key : calc.Keys()) {
usedFields.insert(key.Value());
- }
-
+ }
+
auto processListType = [&](TExprBase typeNode) {
auto structType = typeNode.Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->
Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
@@ -552,7 +552,7 @@ TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node,
usedFields.insert(item->GetName());
}
};
-
+
auto processSortTraits = [&](TExprBase node) {
if (auto maybeSortTraits = node.Maybe<TCoSortTraits>()) {
processListType(maybeSortTraits.Cast().ListType());
@@ -586,7 +586,7 @@ TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node,
TExprNodeList newFrames;
for (const auto& winOnRows : calc.Frames().Ref().ChildrenList()) {
YQL_ENSURE(winOnRows->IsCallable("WinOnRows"));
-
+
TExprNodeList newFrameItems;
newFrameItems.push_back(winOnRows->ChildPtr(0));
@@ -614,15 +614,15 @@ TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node,
for (const auto& item : structType->GetItems()) {
usedFields.insert(item->GetName());
}
- }
-
+ }
+
if (newFrameItems.size() == 1) {
- continue;
- }
-
+ continue;
+ }
+
newFrames.push_back(ctx.ChangeChildren(*winOnRows, std::move(newFrameItems)));
- }
-
+ }
+
newCalcs.emplace_back(
Build<TCoCalcOverWindowTuple>(ctx, calc.Pos())
.Keys(calc.Keys())
@@ -632,30 +632,30 @@ TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node,
.SessionColumns(ctx.NewList(calc.SessionColumns().Pos(), std::move(newSessionColumns)))
.Done().Ptr()
);
- }
-
- // keep input fields
- for (const auto& in : inputStructType->GetItems()) {
+ }
+
+ // keep input fields
+ for (const auto& in : inputStructType->GetItems()) {
if (outMembers.contains(in->GetName()) && !payloadFields.contains(in->GetName())) {
- usedFields.insert(in->GetName());
- }
- }
-
+ usedFields.insert(in->GetName());
+ }
+ }
+
if (usedFields.size() == inputStructType->GetSize() && !dropped) {
- return {};
- }
-
- TExprNode::TListType usedExprList;
- for (const auto& x : usedFields) {
+ return {};
+ }
+
+ TExprNode::TListType usedExprList;
+ for (const auto& x : usedFields) {
usedExprList.push_back(ctx.NewAtom(node->Pos(), x));
- }
-
+ }
+
auto newInput = Build<TCoExtractMembers>(ctx, node->Pos())
.Input(input)
.Members(ctx.NewList(node->Pos(), std::move(usedExprList)))
- .Done()
- .Ptr();
-
+ .Done()
+ .Ptr();
+
auto calcOverWindow = Build<TCoCalcOverWindowGroup>(ctx, node->Pos())
.Input(newInput)
.Calcs(ctx.NewList(node->Pos(), std::move(newCalcs)))
@@ -664,18 +664,18 @@ TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node,
YQL_CLOG(DEBUG, Core) << "Apply ExtractMembers to " << node->Content() << logSuffix;
return Build<TCoExtractMembers>(ctx, node->Pos())
.Input(calcOverWindow)
- .Members(members)
- .Done()
- .Ptr();
-}
-
-TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix) {
- TCoAggregate aggr(node);
- TSet<TStringBuf> outMembers;
- for (const auto& x : members->ChildrenList()) {
- outMembers.insert(x->Content());
- }
-
+ .Members(members)
+ .Done()
+ .Ptr();
+}
+
+TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix) {
+ TCoAggregate aggr(node);
+ TSet<TStringBuf> outMembers;
+ for (const auto& x : members->ChildrenList()) {
+ outMembers.insert(x->Content());
+ }
+
TMaybe<TStringBuf> sessionColumn;
const auto sessionSetting = GetSetting(aggr.Settings().Ref(), "session");
if (sessionSetting) {
@@ -683,54 +683,54 @@ TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, cons
sessionColumn = sessionSetting->Child(1)->Child(0)->Content();
}
- TSet<TStringBuf> usedFields;
+ TSet<TStringBuf> usedFields;
// all actual (non-session) keys will be used
- for (const auto& key : aggr.Keys()) {
+ for (const auto& key : aggr.Keys()) {
if (key.Value() != sessionColumn) {
usedFields.insert(key.Value());
}
- }
-
- TExprNode::TListType newHandlers;
- for (const auto& handler : aggr.Handlers()) {
- if (handler.ColumnName().Ref().IsList()) {
- // many columns
- bool hasColumns = false;
- for (const auto& col : handler.ColumnName().Ref().Children()) {
+ }
+
+ TExprNode::TListType newHandlers;
+ for (const auto& handler : aggr.Handlers()) {
+ if (handler.ColumnName().Ref().IsList()) {
+ // many columns
+ bool hasColumns = false;
+ for (const auto& col : handler.ColumnName().Ref().Children()) {
if (outMembers.contains(col->Content())) {
- hasColumns = true;
- break;
- }
- }
-
- if (!hasColumns) {
- // drop handler
- continue;
- }
- } else {
+ hasColumns = true;
+ break;
+ }
+ }
+
+ if (!hasColumns) {
+ // drop handler
+ continue;
+ }
+ } else {
if (!outMembers.contains(handler.ColumnName().Ref().Content())) {
- // drop handler
- continue;
- }
- }
-
- newHandlers.push_back(handler.Ptr());
- if (handler.DistinctName()) {
- usedFields.insert(handler.DistinctName().Cast().Value());
- } else {
- auto structType = handler.Trait().ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
- for (const auto& item : structType->GetItems()) {
- usedFields.insert(item->GetName());
- }
- }
- }
-
+ // drop handler
+ continue;
+ }
+ }
+
+ newHandlers.push_back(handler.Ptr());
+ if (handler.DistinctName()) {
+ usedFields.insert(handler.DistinctName().Cast().Value());
+ } else {
+ auto structType = handler.Trait().ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
+ for (const auto& item : structType->GetItems()) {
+ usedFields.insert(item->GetName());
+ }
+ }
+ }
+
auto settings = aggr.Settings();
auto hoppingSetting = GetSetting(settings.Ref(), "hopping");
if (hoppingSetting) {
auto traits = TCoHoppingTraits(hoppingSetting->Child(1));
auto timeExtractor = traits.TimeExtractor();
-
+
auto usedType = traits.ItemType().Ref().GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
for (const auto& usedField : usedType->GetItems()) {
usedFields.insert(usedField->GetName());
@@ -739,9 +739,9 @@ TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, cons
TSet<TStringBuf> lambdaSubset;
if (HaveFieldsSubset(timeExtractor.Body().Ptr(), *timeExtractor.Args().Arg(0).Raw(), lambdaSubset, parentsMap)) {
usedFields.insert(lambdaSubset.cbegin(), lambdaSubset.cend());
- }
- }
-
+ }
+ }
+
if (sessionSetting) {
TCoSessionWindowTraits traits(sessionSetting->Child(1)->ChildPtr(1));
@@ -753,36 +753,36 @@ TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, cons
}
}
- auto inputStructType = aggr.Input().Ptr()->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
+ auto inputStructType = aggr.Input().Ptr()->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
YQL_ENSURE(usedFields.size() <= inputStructType->GetSize());
- if (usedFields.size() == inputStructType->GetSize()) {
- return {};
- }
-
- TExprNode::TListType usedExprList;
- for (const auto& x : usedFields) {
- usedExprList.push_back(ctx.NewAtom(aggr.Pos(), x));
- }
-
- auto newInput = Build<TCoExtractMembers>(ctx, aggr.Pos())
- .Input(aggr.Input())
- .Members(ctx.NewList(aggr.Pos(), std::move(usedExprList)))
- .Done()
- .Ptr();
-
+ if (usedFields.size() == inputStructType->GetSize()) {
+ return {};
+ }
+
+ TExprNode::TListType usedExprList;
+ for (const auto& x : usedFields) {
+ usedExprList.push_back(ctx.NewAtom(aggr.Pos(), x));
+ }
+
+ auto newInput = Build<TCoExtractMembers>(ctx, aggr.Pos())
+ .Input(aggr.Input())
+ .Members(ctx.NewList(aggr.Pos(), std::move(usedExprList)))
+ .Done()
+ .Ptr();
+
YQL_CLOG(DEBUG, Core) << "Apply ExtractMembers to " << node->Content() << logSuffix;
- return Build<TCoExtractMembers>(ctx, aggr.Pos())
- .Input<TCoAggregate>()
- .Input(newInput)
- .Keys(aggr.Keys())
- .Handlers(ctx.NewList(aggr.Pos(), std::move(newHandlers)))
- .Settings(aggr.Settings())
- .Build()
- .Members(members)
- .Done()
- .Ptr();
-}
-
+ return Build<TCoExtractMembers>(ctx, aggr.Pos())
+ .Input<TCoAggregate>()
+ .Input(newInput)
+ .Keys(aggr.Keys())
+ .Handlers(ctx.NewList(aggr.Pos(), std::move(newHandlers)))
+ .Settings(aggr.Settings())
+ .Build()
+ .Members(members)
+ .Done()
+ .Ptr();
+}
+
TExprNode::TPtr ApplyExtractMembersToCollect(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix) {
TCoCollect collect(node);
YQL_CLOG(DEBUG, Core) << "Move ExtractMembers over " << node->Content() << logSuffix;
@@ -794,4 +794,4 @@ TExprNode::TPtr ApplyExtractMembersToCollect(const TExprNode::TPtr& node, const
.Done().Ptr();
}
-} // NYql
+} // NYql
diff --git a/ydb/library/yql/core/common_opt/yql_co_extr_members.h b/ydb/library/yql/core/common_opt/yql_co_extr_members.h
index 8244636784..79ce5fd098 100644
--- a/ydb/library/yql/core/common_opt/yql_co_extr_members.h
+++ b/ydb/library/yql/core/common_opt/yql_co_extr_members.h
@@ -1,29 +1,29 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
-
+
#include <ydb/library/yql/ast/yql_expr.h>
-
-#include <util/generic/strbuf.h>
-
-
-namespace NYql {
-
-TExprNode::TPtr ApplyExtractMembersToTake(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToSkip(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToExtend(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToSkipNullMembers(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToFilterNullMembers(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToSort(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToAssumeUnique(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToTop(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToEquiJoin(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToFlatMap(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToPartitionByKey(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix);
+
+#include <util/generic/strbuf.h>
+
+
+namespace NYql {
+
+TExprNode::TPtr ApplyExtractMembersToTake(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToSkip(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToExtend(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToSkipNullMembers(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToFilterNullMembers(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToSort(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToAssumeUnique(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToTop(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToEquiJoin(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToFlatMap(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToPartitionByKey(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToCalcOverWindow(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
+TExprNode::TPtr ApplyExtractMembersToAggregate(const TExprNode::TPtr& node, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx, TStringBuf logSuffix);
TExprNode::TPtr ApplyExtractMembersToChopper(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
TExprNode::TPtr ApplyExtractMembersToCollect(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
TExprNode::TPtr ApplyExtractMembersToMapJoinCore(const TExprNode::TPtr& node, const TExprNode::TPtr& members, TExprContext& ctx, TStringBuf logSuffix);
-
-} // NYql
+
+} // NYql
diff --git a/ydb/library/yql/core/common_opt/yql_co_finalizers.cpp b/ydb/library/yql/core/common_opt/yql_co_finalizers.cpp
index b77766bc6f..9013511f95 100644
--- a/ydb/library/yql/core/common_opt/yql_co_finalizers.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_finalizers.cpp
@@ -1,192 +1,192 @@
-#include "yql_co_extr_members.h"
-#include "yql_co.h"
-
+#include "yql_co_extr_members.h"
+#include "yql_co.h"
+
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
#include <ydb/library/yql/core/yql_opt_utils.h>
-
+
#include <ydb/library/yql/utils/log/log.h>
-
-namespace NYql {
-
-namespace {
-
-using namespace NNodes;
-
-void SubsetFieldsForNodeWithMultiUsage(const TExprNode::TPtr& node, const TParentsMap& parentsMap,
- TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx,
- std::function<TExprNode::TPtr(const TExprNode::TPtr&, const TExprNode::TPtr&, const TParentsMap&, TExprContext&)> handler)
-{
- // Ignore stream input, because it cannot be used multiple times
- if (node->GetTypeAnn()->GetKind() != ETypeAnnotationKind::List) {
- return;
- }
- auto itemType = node->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
- if (itemType->GetKind() != ETypeAnnotationKind::Struct) {
- return;
- }
- auto structType = itemType->Cast<TStructExprType>();
-
- auto it = parentsMap.find(node.Get());
- if (it == parentsMap.cend() || it->second.size() <= 1) {
- return;
- }
-
- TSet<TStringBuf> usedFields;
- for (auto parent: it->second) {
- if (auto maybeFlatMap = TMaybeNode<TCoFlatMapBase>(parent)) {
- auto flatMap = maybeFlatMap.Cast();
- TSet<TStringBuf> lambdaSubset;
- if (!HaveFieldsSubset(flatMap.Lambda().Body().Ptr(), flatMap.Lambda().Args().Arg(0).Ref(), lambdaSubset, parentsMap)) {
- return;
- }
- usedFields.insert(lambdaSubset.cbegin(), lambdaSubset.cend());
- }
- else if (auto maybeExtractMembers = TMaybeNode<TCoExtractMembers>(parent)) {
- auto extractMembers = maybeExtractMembers.Cast();
- for (auto member: extractMembers.Members()) {
- usedFields.insert(member.Value());
- }
- }
- else {
- return;
- }
- if (usedFields.size() == structType->GetSize()) {
- return;
- }
- }
-
- TExprNode::TListType members;
- for (auto column : usedFields) {
- members.push_back(ctx.NewAtom(node->Pos(), column));
- }
-
- auto newInput = handler(node, ctx.NewList(node->Pos(), std::move(members)), parentsMap, ctx);
- if (!newInput || newInput == node) {
- return;
- }
-
- for (auto parent: it->second) {
- if (TCoExtractMembers::Match(parent)) {
- if (parent->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>()->GetSize() == usedFields.size()) {
- toOptimize[parent] = newInput;
- } else {
- toOptimize[parent] = ctx.ChangeChild(*parent, 0, TExprNode::TPtr(newInput));
- }
- } else {
- toOptimize[parent] = ctx.Builder(parent->Pos())
- .Callable(parent->Content())
- .Add(0, newInput)
- .Lambda(1)
- .Param("item")
- .Apply(parent->ChildPtr(1)).With(0, "item").Seal()
- .Seal()
- .Seal()
- .Build();
- }
- }
-}
-
-}
-
-void RegisterCoFinalizers(TFinalizingOptimizerMap& map) {
- map[TCoExtend::CallableName()] = map[TCoOrderedExtend::CallableName()] = map[TCoMerge::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
- return ApplyExtractMembersToExtend(input, members, ctx, " with multi-usage");
- }
- );
- };
-
- map[TCoTake::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
- return ApplyExtractMembersToTake(input, members, ctx, " with multi-usage");
- }
- );
- };
-
- map[TCoSkip::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
- return ApplyExtractMembersToSkip(input, members, ctx, " with multi-usage");
- }
- );
- };
-
- map[TCoSkipNullMembers::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
- return ApplyExtractMembersToSkipNullMembers(input, members, ctx, " with multi-usage");
- }
- );
- };
-
- map[TCoFlatMap::CallableName()] = map[TCoOrderedFlatMap::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
- return ApplyExtractMembersToFlatMap(input, members, ctx, " with multi-usage");
- }
- );
- };
-
- map[TCoSort::CallableName()] = map[TCoAssumeSorted::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx) {
- return ApplyExtractMembersToSort(input, members, parentsMap, ctx, " with multi-usage");
- }
- );
- };
-
- map[TCoAssumeUnique::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
- return ApplyExtractMembersToAssumeUnique(input, members, ctx, " with multi-usage");
- }
- );
- };
-
- map[TCoTop::CallableName()] = map[TCoTopSort::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx) {
- return ApplyExtractMembersToTop(input, members, parentsMap, ctx, " with multi-usage");
- }
- );
- };
-
- map[TCoEquiJoin::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
- return ApplyExtractMembersToEquiJoin(input, members, ctx, " with multi-usage");
- }
- );
- };
-
- map[TCoPartitionByKey::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
- return ApplyExtractMembersToPartitionByKey(input, members, ctx, " with multi-usage");
- }
- );
- };
-
+
+namespace NYql {
+
+namespace {
+
+using namespace NNodes;
+
+void SubsetFieldsForNodeWithMultiUsage(const TExprNode::TPtr& node, const TParentsMap& parentsMap,
+ TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx,
+ std::function<TExprNode::TPtr(const TExprNode::TPtr&, const TExprNode::TPtr&, const TParentsMap&, TExprContext&)> handler)
+{
+ // Ignore stream input, because it cannot be used multiple times
+ if (node->GetTypeAnn()->GetKind() != ETypeAnnotationKind::List) {
+ return;
+ }
+ auto itemType = node->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ if (itemType->GetKind() != ETypeAnnotationKind::Struct) {
+ return;
+ }
+ auto structType = itemType->Cast<TStructExprType>();
+
+ auto it = parentsMap.find(node.Get());
+ if (it == parentsMap.cend() || it->second.size() <= 1) {
+ return;
+ }
+
+ TSet<TStringBuf> usedFields;
+ for (auto parent: it->second) {
+ if (auto maybeFlatMap = TMaybeNode<TCoFlatMapBase>(parent)) {
+ auto flatMap = maybeFlatMap.Cast();
+ TSet<TStringBuf> lambdaSubset;
+ if (!HaveFieldsSubset(flatMap.Lambda().Body().Ptr(), flatMap.Lambda().Args().Arg(0).Ref(), lambdaSubset, parentsMap)) {
+ return;
+ }
+ usedFields.insert(lambdaSubset.cbegin(), lambdaSubset.cend());
+ }
+ else if (auto maybeExtractMembers = TMaybeNode<TCoExtractMembers>(parent)) {
+ auto extractMembers = maybeExtractMembers.Cast();
+ for (auto member: extractMembers.Members()) {
+ usedFields.insert(member.Value());
+ }
+ }
+ else {
+ return;
+ }
+ if (usedFields.size() == structType->GetSize()) {
+ return;
+ }
+ }
+
+ TExprNode::TListType members;
+ for (auto column : usedFields) {
+ members.push_back(ctx.NewAtom(node->Pos(), column));
+ }
+
+ auto newInput = handler(node, ctx.NewList(node->Pos(), std::move(members)), parentsMap, ctx);
+ if (!newInput || newInput == node) {
+ return;
+ }
+
+ for (auto parent: it->second) {
+ if (TCoExtractMembers::Match(parent)) {
+ if (parent->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>()->GetSize() == usedFields.size()) {
+ toOptimize[parent] = newInput;
+ } else {
+ toOptimize[parent] = ctx.ChangeChild(*parent, 0, TExprNode::TPtr(newInput));
+ }
+ } else {
+ toOptimize[parent] = ctx.Builder(parent->Pos())
+ .Callable(parent->Content())
+ .Add(0, newInput)
+ .Lambda(1)
+ .Param("item")
+ .Apply(parent->ChildPtr(1)).With(0, "item").Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+ }
+}
+
+}
+
+void RegisterCoFinalizers(TFinalizingOptimizerMap& map) {
+ map[TCoExtend::CallableName()] = map[TCoOrderedExtend::CallableName()] = map[TCoMerge::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
+ return ApplyExtractMembersToExtend(input, members, ctx, " with multi-usage");
+ }
+ );
+ };
+
+ map[TCoTake::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
+ return ApplyExtractMembersToTake(input, members, ctx, " with multi-usage");
+ }
+ );
+ };
+
+ map[TCoSkip::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
+ return ApplyExtractMembersToSkip(input, members, ctx, " with multi-usage");
+ }
+ );
+ };
+
+ map[TCoSkipNullMembers::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
+ return ApplyExtractMembersToSkipNullMembers(input, members, ctx, " with multi-usage");
+ }
+ );
+ };
+
+ map[TCoFlatMap::CallableName()] = map[TCoOrderedFlatMap::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
+ return ApplyExtractMembersToFlatMap(input, members, ctx, " with multi-usage");
+ }
+ );
+ };
+
+ map[TCoSort::CallableName()] = map[TCoAssumeSorted::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx) {
+ return ApplyExtractMembersToSort(input, members, parentsMap, ctx, " with multi-usage");
+ }
+ );
+ };
+
+ map[TCoAssumeUnique::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
+ return ApplyExtractMembersToAssumeUnique(input, members, ctx, " with multi-usage");
+ }
+ );
+ };
+
+ map[TCoTop::CallableName()] = map[TCoTopSort::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx) {
+ return ApplyExtractMembersToTop(input, members, parentsMap, ctx, " with multi-usage");
+ }
+ );
+ };
+
+ map[TCoEquiJoin::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
+ return ApplyExtractMembersToEquiJoin(input, members, ctx, " with multi-usage");
+ }
+ );
+ };
+
+ map[TCoPartitionByKey::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
+ return ApplyExtractMembersToPartitionByKey(input, members, ctx, " with multi-usage");
+ }
+ );
+ };
+
map[TCoCalcOverWindowGroup::CallableName()] = map[TCoCalcOverWindow::CallableName()] =
map[TCoCalcOverSessionWindow::CallableName()] =
[](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx)
{
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
- return ApplyExtractMembersToCalcOverWindow(input, members, ctx, " with multi-usage");
- }
- );
- };
-
- map[TCoAggregate::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
- SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
- [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx) {
- return ApplyExtractMembersToAggregate(input, members, parentsMap, ctx, " with multi-usage");
- }
- );
- };
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap&, TExprContext& ctx) {
+ return ApplyExtractMembersToCalcOverWindow(input, members, ctx, " with multi-usage");
+ }
+ );
+ };
+
+ map[TCoAggregate::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
+ SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
+ [] (const TExprNode::TPtr& input, const TExprNode::TPtr& members, const TParentsMap& parentsMap, TExprContext& ctx) {
+ return ApplyExtractMembersToAggregate(input, members, parentsMap, ctx, " with multi-usage");
+ }
+ );
+ };
map[TCoChopper::CallableName()] = [](const TExprNode::TPtr& node, TNodeOnNodeOwnedMap& toOptimize, TExprContext& ctx, TOptimizeContext& optCtx) {
SubsetFieldsForNodeWithMultiUsage(node, *optCtx.ParentsMap, toOptimize, ctx,
@@ -203,6 +203,6 @@ void RegisterCoFinalizers(TFinalizingOptimizerMap& map) {
}
);
};
-}
-
-} // NYql
+}
+
+} // NYql
diff --git a/ydb/library/yql/core/common_opt/yql_co_flow1.cpp b/ydb/library/yql/core/common_opt/yql_co_flow1.cpp
index 728838dc8e..b2b4eb0760 100644
--- a/ydb/library/yql/core/common_opt/yql_co_flow1.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_flow1.cpp
@@ -7,27 +7,27 @@
#include <ydb/library/yql/utils/log/log.h>
-#include <util/generic/hash.h>
-#include <util/generic/string.h>
-#include <util/generic/set.h>
-#include <util/string/cast.h>
-
-#include <utility>
-
+#include <util/generic/hash.h>
+#include <util/generic/string.h>
+#include <util/generic/set.h>
+#include <util/string/cast.h>
+
+#include <utility>
+
namespace NYql {
namespace {
using namespace NNodes;
-bool IsConstMapLambda(TCoLambda lambda) {
+bool IsConstMapLambda(TCoLambda lambda) {
const auto body = lambda.Body();
return body.Ref().IsCallable("Just") && body.Ref().GetDependencyScope()->second != lambda.Raw();
}
-template <typename TResult>
-TExprNode::TPtr FuseFlatmaps(TCoFlatMapBase outerMap, TExprContext& ctx, TTypeAnnotationContext* types) {
- auto innerMap = outerMap.Input().template Cast<TCoFlatMapBase>();
+template <typename TResult>
+TExprNode::TPtr FuseFlatmaps(TCoFlatMapBase outerMap, TExprContext& ctx, TTypeAnnotationContext* types) {
+ auto innerMap = outerMap.Input().template Cast<TCoFlatMapBase>();
auto innerBody = innerMap.Lambda().Body();
auto outerBody = outerMap.Lambda().Body();
auto outerLambda = outerMap.Lambda().Ptr();
@@ -68,28 +68,28 @@ TExprNode::TPtr FuseFlatmaps(TCoFlatMapBase outerMap, TExprContext& ctx, TTypeAn
if (IsJustOrSingleAsList(innerBody.Ref())) {
- auto placeHolder = ctx.NewArgument(outerMap.Pos(), "placeholder");
-
- auto status = OptimizeExpr(outerLambda, outerLambda, [&placeHolder, outerLambdaArg](const TExprNode::TPtr& node, TExprContext&) -> TExprNode::TPtr {
- if (TCoDependsOn::Match(node.Get()) && node->Child(0) == outerLambdaArg) {
- return placeHolder;
- }
- return node;
- }, ctx, TOptimizeExprSettings(types));
-
- if (status.Level == IGraphTransformer::TStatus::Error) {
- return {};
- }
-
- auto outerArgValue = innerBody.template Maybe<TCoJust>()
- ? innerBody.template Cast<TCoJust>().Input()
- : innerBody.template Cast<TCoAsList>().Arg(0);
-
- YQL_CLOG(DEBUG, Core) << "FuseFlatmaps with inner " << innerBody.Ref().Content();
-
- if (outerMap.Input().Ref().GetTypeAnn()->GetKind() != innerMap.Input().Ref().GetTypeAnn()->GetKind()
- && innerMap.Input().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
-
+ auto placeHolder = ctx.NewArgument(outerMap.Pos(), "placeholder");
+
+ auto status = OptimizeExpr(outerLambda, outerLambda, [&placeHolder, outerLambdaArg](const TExprNode::TPtr& node, TExprContext&) -> TExprNode::TPtr {
+ if (TCoDependsOn::Match(node.Get()) && node->Child(0) == outerLambdaArg) {
+ return placeHolder;
+ }
+ return node;
+ }, ctx, TOptimizeExprSettings(types));
+
+ if (status.Level == IGraphTransformer::TStatus::Error) {
+ return {};
+ }
+
+ auto outerArgValue = innerBody.template Maybe<TCoJust>()
+ ? innerBody.template Cast<TCoJust>().Input()
+ : innerBody.template Cast<TCoAsList>().Arg(0);
+
+ YQL_CLOG(DEBUG, Core) << "FuseFlatmaps with inner " << innerBody.Ref().Content();
+
+ if (outerMap.Input().Ref().GetTypeAnn()->GetKind() != innerMap.Input().Ref().GetTypeAnn()->GetKind()
+ && innerMap.Input().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
+
if (outerMap.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow &&
outerBody.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Flow) {
@@ -111,115 +111,115 @@ TExprNode::TPtr FuseFlatmaps(TCoFlatMapBase outerMap, TExprContext& ctx, TTypeAn
.Done().Ptr();
} else if (outerMap.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream &&
- outerBody.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Stream) {
-
- return Build<TResult>(ctx, outerMap.Pos())
- .Input(innerMap.Input())
- .Lambda()
- .Args({"item"})
- .template Body<TCoToStream>()
- .template Input<TExprApplier>()
- .Apply(TCoLambda(outerLambda))
- .With(0, outerArgValue)
- .With(innerMap.Lambda().Args().Arg(0), "item")
- .template With<TCoDependsOn>(TExprBase(placeHolder))
- .Input("item")
- .Build()
- .Build()
- .FreeArgs()
- .template Add<TCoDependsOn>()
- .Input("item")
- .Build()
- .Build()
- .Build()
- .Build()
- .Done().Ptr();
-
- } else if (outerMap.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List &&
- outerBody.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
-
- return Build<TResult>(ctx, outerMap.Pos())
- .Input(innerMap.Input())
- .Lambda()
- .Args({"item"})
- .template Body<TCoToList>()
- .template Optional<TExprApplier>()
- .Apply(TCoLambda(outerLambda))
- .With(0, outerArgValue)
- .With(innerMap.Lambda().Args().Arg(0), "item")
- .template With<TCoDependsOn>(TExprBase(placeHolder))
- .Input("item")
- .Build()
- .Build()
- .Build()
- .Build()
- .Done().Ptr();
- }
- }
-
+ outerBody.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Stream) {
+
+ return Build<TResult>(ctx, outerMap.Pos())
+ .Input(innerMap.Input())
+ .Lambda()
+ .Args({"item"})
+ .template Body<TCoToStream>()
+ .template Input<TExprApplier>()
+ .Apply(TCoLambda(outerLambda))
+ .With(0, outerArgValue)
+ .With(innerMap.Lambda().Args().Arg(0), "item")
+ .template With<TCoDependsOn>(TExprBase(placeHolder))
+ .Input("item")
+ .Build()
+ .Build()
+ .FreeArgs()
+ .template Add<TCoDependsOn>()
+ .Input("item")
+ .Build()
+ .Build()
+ .Build()
+ .Build()
+ .Done().Ptr();
+
+ } else if (outerMap.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List &&
+ outerBody.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
+
+ return Build<TResult>(ctx, outerMap.Pos())
+ .Input(innerMap.Input())
+ .Lambda()
+ .Args({"item"})
+ .template Body<TCoToList>()
+ .template Optional<TExprApplier>()
+ .Apply(TCoLambda(outerLambda))
+ .With(0, outerArgValue)
+ .With(innerMap.Lambda().Args().Arg(0), "item")
+ .template With<TCoDependsOn>(TExprBase(placeHolder))
+ .Input("item")
+ .Build()
+ .Build()
+ .Build()
+ .Build()
+ .Done().Ptr();
+ }
+ }
+
return Build<TResult>(ctx, outerMap.Pos())
.Input(innerMap.Input())
.Lambda()
.Args({"item"})
.template Body<TExprApplier>()
- .Apply(TCoLambda(outerLambda))
- .With(0, outerArgValue)
- .With(innerMap.Lambda().Args().Arg(0), "item")
- .template With<TCoDependsOn>(TExprBase(placeHolder))
- .Input("item")
+ .Apply(TCoLambda(outerLambda))
+ .With(0, outerArgValue)
+ .With(innerMap.Lambda().Args().Arg(0), "item")
+ .template With<TCoDependsOn>(TExprBase(placeHolder))
+ .Input("item")
.Build()
.Build()
- .Build()
- .Done().Ptr();
+ .Build()
+ .Done().Ptr();
}
- if (innerBody.template Maybe<TCoOptionalIf>() || innerBody.template Maybe<TCoListIf>()) {
- auto conditional = innerBody.template Cast<TCoConditionalValueBase>();
-
- auto placeHolder = ctx.NewArgument(outerMap.Pos(), "placeholder");
- auto outerLambda = outerMap.Lambda().Ptr();
- auto outerLambdaArg = outerMap.Lambda().Args().Arg(0).Raw();
-
- auto status = OptimizeExpr(outerLambda, outerLambda, [&placeHolder, outerLambdaArg](const TExprNode::TPtr& node, TExprContext&) -> TExprNode::TPtr {
- if (TCoDependsOn::Match(node.Get()) && node->Child(0) == outerLambdaArg) {
- return placeHolder;
- }
- return node;
- }, ctx, TOptimizeExprSettings(types));
-
- if (status.Level == IGraphTransformer::TStatus::Error) {
- return {};
- }
-
- auto value = ctx.Builder(outerMap.Pos())
- .Apply(outerLambda)
- .With(0, conditional.Value().Ptr())
- .Seal()
- .Build();
-
+ if (innerBody.template Maybe<TCoOptionalIf>() || innerBody.template Maybe<TCoListIf>()) {
+ auto conditional = innerBody.template Cast<TCoConditionalValueBase>();
+
+ auto placeHolder = ctx.NewArgument(outerMap.Pos(), "placeholder");
+ auto outerLambda = outerMap.Lambda().Ptr();
+ auto outerLambdaArg = outerMap.Lambda().Args().Arg(0).Raw();
+
+ auto status = OptimizeExpr(outerLambda, outerLambda, [&placeHolder, outerLambdaArg](const TExprNode::TPtr& node, TExprContext&) -> TExprNode::TPtr {
+ if (TCoDependsOn::Match(node.Get()) && node->Child(0) == outerLambdaArg) {
+ return placeHolder;
+ }
+ return node;
+ }, ctx, TOptimizeExprSettings(types));
+
+ if (status.Level == IGraphTransformer::TStatus::Error) {
+ return {};
+ }
+
+ auto value = ctx.Builder(outerMap.Pos())
+ .Apply(outerLambda)
+ .With(0, conditional.Value().Ptr())
+ .Seal()
+ .Build();
+
if (outerBody.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream || outerBody.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow) {
- value = ctx.Builder(outerMap.Pos())
- .Callable(TCoForwardList::CallableName())
- .Add(0, value)
- .Seal()
- .Build();
- }
-
- auto conditionName = outerBody.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional
- ? TCoFlatOptionalIf::CallableName()
- : TCoFlatListIf::CallableName();
-
- auto newBody = Build<TCoConditionalValueBase>(ctx, outerMap.Pos())
- .CallableName(conditionName)
- .Predicate(conditional.Predicate())
- .Value(value)
- .Done();
-
- YQL_CLOG(DEBUG, Core) << "FuseFlatmaps with inner " << innerBody.Ref().Content();
-
- if (outerMap.Input().Ref().GetTypeAnn()->GetKind() != innerMap.Input().Ref().GetTypeAnn()->GetKind()
- && innerMap.Input().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
-
+ value = ctx.Builder(outerMap.Pos())
+ .Callable(TCoForwardList::CallableName())
+ .Add(0, value)
+ .Seal()
+ .Build();
+ }
+
+ auto conditionName = outerBody.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional
+ ? TCoFlatOptionalIf::CallableName()
+ : TCoFlatListIf::CallableName();
+
+ auto newBody = Build<TCoConditionalValueBase>(ctx, outerMap.Pos())
+ .CallableName(conditionName)
+ .Predicate(conditional.Predicate())
+ .Value(value)
+ .Done();
+
+ YQL_CLOG(DEBUG, Core) << "FuseFlatmaps with inner " << innerBody.Ref().Content();
+
+ if (outerMap.Input().Ref().GetTypeAnn()->GetKind() != innerMap.Input().Ref().GetTypeAnn()->GetKind()
+ && innerMap.Input().Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
+
if (outerMap.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow &&
outerBody.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Flow) {
@@ -240,183 +240,183 @@ TExprNode::TPtr FuseFlatmaps(TCoFlatMapBase outerMap, TExprContext& ctx, TTypeAn
.Done().Ptr();
} else if (outerMap.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream &&
- outerBody.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Stream) {
-
- return Build<TResult>(ctx, outerMap.Pos())
- .Input(innerMap.Input())
- .Lambda()
- .Args({"item"})
- .template Body<TCoToStream>()
- .template Input<TExprApplier>()
- .Apply(newBody)
- .With(innerMap.Lambda().Args().Arg(0), "item")
- .template With<TCoDependsOn>(TExprBase(placeHolder))
- .Input("item")
- .Build()
- .Build()
- .FreeArgs()
- .template Add<TCoDependsOn>()
- .Input("item")
- .Build()
- .Build()
- .Build()
- .Build()
- .Done().Ptr();
-
- } else if (outerMap.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List &&
- outerBody.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
-
- return Build<TResult>(ctx, outerMap.Pos())
- .Input(innerMap.Input())
- .Lambda()
- .Args({"item"})
- .template Body<TCoToList>()
- .template Optional<TExprApplier>()
- .Apply(newBody)
- .With(innerMap.Lambda().Args().Arg(0), "item")
- .template With<TCoDependsOn>(TExprBase(placeHolder))
- .Input("item")
- .Build()
- .Build()
- .Build()
- .Build()
- .Done().Ptr();
- }
- }
-
+ outerBody.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Stream) {
+
+ return Build<TResult>(ctx, outerMap.Pos())
+ .Input(innerMap.Input())
+ .Lambda()
+ .Args({"item"})
+ .template Body<TCoToStream>()
+ .template Input<TExprApplier>()
+ .Apply(newBody)
+ .With(innerMap.Lambda().Args().Arg(0), "item")
+ .template With<TCoDependsOn>(TExprBase(placeHolder))
+ .Input("item")
+ .Build()
+ .Build()
+ .FreeArgs()
+ .template Add<TCoDependsOn>()
+ .Input("item")
+ .Build()
+ .Build()
+ .Build()
+ .Build()
+ .Done().Ptr();
+
+ } else if (outerMap.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::List &&
+ outerBody.Ref().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
+
+ return Build<TResult>(ctx, outerMap.Pos())
+ .Input(innerMap.Input())
+ .Lambda()
+ .Args({"item"})
+ .template Body<TCoToList>()
+ .template Optional<TExprApplier>()
+ .Apply(newBody)
+ .With(innerMap.Lambda().Args().Arg(0), "item")
+ .template With<TCoDependsOn>(TExprBase(placeHolder))
+ .Input("item")
+ .Build()
+ .Build()
+ .Build()
+ .Build()
+ .Done().Ptr();
+ }
+ }
+
return Build<TResult>(ctx, outerMap.Pos())
.Input(innerMap.Input())
.Lambda()
.Args({"item"})
.template Body<TExprApplier>()
- .Apply(newBody)
- .With(innerMap.Lambda().Args().Arg(0), "item")
- .template With<TCoDependsOn>(TExprBase(placeHolder))
- .Input("item")
+ .Apply(newBody)
+ .With(innerMap.Lambda().Args().Arg(0), "item")
+ .template With<TCoDependsOn>(TExprBase(placeHolder))
+ .Input("item")
.Build()
.Build()
- .Build()
- .Done().Ptr();
- }
-
- if (innerBody.template Maybe<TCoVisit>() && outerBody.template Maybe<TCoJust>()) {
- auto outerLambda = outerMap.Lambda().Ptr();
-
- auto originalVisit = innerBody.Ptr();
- YQL_CLOG(DEBUG, Core) << "FuseFlatmaps with inner " << innerBody.Ref().Content();
- return ctx.Builder(outerMap.Pos())
- .Callable(TResult::CallableName())
- .Add(0, innerMap.Input().Ptr())
- .Lambda(1)
- .Param("item")
- .Callable("Visit")
- .ApplyPartial(0, innerMap.Lambda().Args().Ptr(), originalVisit->HeadPtr())
- .With(0, "item")
- .Seal()
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- for (size_t i = 1; i < originalVisit->ChildrenSize(); ++i) {
- auto child = originalVisit->ChildPtr(i);
- if (child->IsAtom()) {
- auto lambda = originalVisit->Child(i + 1);
- parent
- .Add(i, std::move(child))
- .Lambda(i + 1)
- .Param("visitItem")
- .Callable(TResult::CallableName())
- .Apply(0, lambda)
- .With(0, "visitItem")
- .Seal()
- .Lambda(1)
- .Param("mapItem")
- .Apply(outerLambda)
- .With(0, "mapItem")
- .Seal()
- .Seal()
- .Seal()
- .Seal();
- ++i;
- }
- else {
- parent.Callable(i, TResult::CallableName())
- .Add(0, std::move(child))
- .Lambda(1)
- .Param("mapItem")
- .Apply(outerLambda)
- .With(0, "mapItem")
- .Seal()
- .Seal()
- .Seal();
- }
- }
- return parent;
- })
- .Seal()
- .Seal()
- .Seal()
- .Build();
- }
-
- if (innerBody.template Maybe<TCoAsList>() && outerBody.template Maybe<TCoJust>()) {
-
- auto outerJustInput = outerBody.template Cast<TCoJust>().Input().Ptr();
- auto outerLambdaArgs = outerMap.Lambda().Args().Ptr();
- auto innerLambdaArg = innerMap.Lambda().Args().Arg(0).Raw();
-
- auto originalAsList = innerBody.Ptr();
-
- YQL_CLOG(DEBUG, Core) << "FuseFlatmaps with inner " << innerBody.Ref().Content();
- return ctx.Builder(outerMap.Pos())
- .Callable(TResult::CallableName())
- .Add(0, innerMap.Input().Ptr())
- .Lambda(1)
- .Param("item")
- .Callable("AsList")
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- for (size_t i = 0; i < originalAsList->ChildrenSize(); ++i) {
- parent.ApplyPartial(i, outerLambdaArgs, outerJustInput)
- .With(0, originalAsList->ChildPtr(i))
- .WithNode(*innerLambdaArg, "item")
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Seal()
- .Seal()
- .Build();
- }
-
- if (innerBody.template Maybe<TCoExtendBase>() && outerBody.template Maybe<TCoJust>()) {
-
- auto originalExtend = innerBody.Ptr();
-
- YQL_CLOG(DEBUG, Core) << "FuseFlatmaps with inner " << innerBody.Ref().Content();
- return ctx.Builder(outerMap.Pos())
- .Callable(TResult::CallableName())
- .Add(0, innerMap.Input().Ptr())
- .Lambda(1)
- .Param("item")
- .Callable(originalExtend->Content())
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- for (size_t i = 0; i < originalExtend->ChildrenSize(); ++i) {
- parent.ApplyPartial(i, {}, outerMap.Ptr())
- .WithNode(outerMap.Input().Ref(), originalExtend->ChildPtr(i))
- .WithNode(innerMap.Lambda().Args().Arg(0).Ref(), "item")
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Seal()
- .Seal()
- .Build();
+ .Build()
+ .Done().Ptr();
}
+ if (innerBody.template Maybe<TCoVisit>() && outerBody.template Maybe<TCoJust>()) {
+ auto outerLambda = outerMap.Lambda().Ptr();
+
+ auto originalVisit = innerBody.Ptr();
+ YQL_CLOG(DEBUG, Core) << "FuseFlatmaps with inner " << innerBody.Ref().Content();
+ return ctx.Builder(outerMap.Pos())
+ .Callable(TResult::CallableName())
+ .Add(0, innerMap.Input().Ptr())
+ .Lambda(1)
+ .Param("item")
+ .Callable("Visit")
+ .ApplyPartial(0, innerMap.Lambda().Args().Ptr(), originalVisit->HeadPtr())
+ .With(0, "item")
+ .Seal()
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ for (size_t i = 1; i < originalVisit->ChildrenSize(); ++i) {
+ auto child = originalVisit->ChildPtr(i);
+ if (child->IsAtom()) {
+ auto lambda = originalVisit->Child(i + 1);
+ parent
+ .Add(i, std::move(child))
+ .Lambda(i + 1)
+ .Param("visitItem")
+ .Callable(TResult::CallableName())
+ .Apply(0, lambda)
+ .With(0, "visitItem")
+ .Seal()
+ .Lambda(1)
+ .Param("mapItem")
+ .Apply(outerLambda)
+ .With(0, "mapItem")
+ .Seal()
+ .Seal()
+ .Seal()
+ .Seal();
+ ++i;
+ }
+ else {
+ parent.Callable(i, TResult::CallableName())
+ .Add(0, std::move(child))
+ .Lambda(1)
+ .Param("mapItem")
+ .Apply(outerLambda)
+ .With(0, "mapItem")
+ .Seal()
+ .Seal()
+ .Seal();
+ }
+ }
+ return parent;
+ })
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+
+ if (innerBody.template Maybe<TCoAsList>() && outerBody.template Maybe<TCoJust>()) {
+
+ auto outerJustInput = outerBody.template Cast<TCoJust>().Input().Ptr();
+ auto outerLambdaArgs = outerMap.Lambda().Args().Ptr();
+ auto innerLambdaArg = innerMap.Lambda().Args().Arg(0).Raw();
+
+ auto originalAsList = innerBody.Ptr();
+
+ YQL_CLOG(DEBUG, Core) << "FuseFlatmaps with inner " << innerBody.Ref().Content();
+ return ctx.Builder(outerMap.Pos())
+ .Callable(TResult::CallableName())
+ .Add(0, innerMap.Input().Ptr())
+ .Lambda(1)
+ .Param("item")
+ .Callable("AsList")
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ for (size_t i = 0; i < originalAsList->ChildrenSize(); ++i) {
+ parent.ApplyPartial(i, outerLambdaArgs, outerJustInput)
+ .With(0, originalAsList->ChildPtr(i))
+ .WithNode(*innerLambdaArg, "item")
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+
+ if (innerBody.template Maybe<TCoExtendBase>() && outerBody.template Maybe<TCoJust>()) {
+
+ auto originalExtend = innerBody.Ptr();
+
+ YQL_CLOG(DEBUG, Core) << "FuseFlatmaps with inner " << innerBody.Ref().Content();
+ return ctx.Builder(outerMap.Pos())
+ .Callable(TResult::CallableName())
+ .Add(0, innerMap.Input().Ptr())
+ .Lambda(1)
+ .Param("item")
+ .Callable(originalExtend->Content())
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ for (size_t i = 0; i < originalExtend->ChildrenSize(); ++i) {
+ parent.ApplyPartial(i, {}, outerMap.Ptr())
+ .WithNode(outerMap.Input().Ref(), originalExtend->ChildPtr(i))
+ .WithNode(innerMap.Lambda().Args().Arg(0).Ref(), "item")
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+
return outerMap.Ptr();
}
-TExprNode::TPtr FuseCombineByKeyOverFlatmap(TCoCombineByKey combine, TExprContext& ctx) {
- auto flatmap = combine.Input().Cast<TCoFlatMapBase>();
+TExprNode::TPtr FuseCombineByKeyOverFlatmap(TCoCombineByKey combine, TExprContext& ctx) {
+ auto flatmap = combine.Input().Cast<TCoFlatMapBase>();
auto inputType = flatmap.Input().Ref().GetTypeAnn();
if (inputType->GetKind() == ETypeAnnotationKind::Optional) {
@@ -434,7 +434,7 @@ TExprNode::TPtr FuseCombineByKeyOverFlatmap(TCoCombineByKey combine, TExprContex
? flatmapBody.Cast<TCoJust>().Input()
: flatmapBody.Cast<TCoAsList>().Arg(0);
- auto ret = Build<TCoCombineByKey>(ctx, combine.Pos())
+ auto ret = Build<TCoCombineByKey>(ctx, combine.Pos())
.Input(flatmap.Input())
.PreMapLambda()
.Args({"item"})
@@ -927,8 +927,8 @@ TExprNode::TPtr ConvertFoldBySumToLength(const TExprNode::TPtr& node, TExprConte
auto arg1 = lambda.Tail().Child(0);
const bool isInc = lambda.Tail().IsCallable("Inc");
auto arg2 = isInc ? nullptr : lambda.Tail().Child(1);
- if (arg1->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data ||
- (arg2 && arg2->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data)) {
+ if (arg1->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data ||
+ (arg2 && arg2->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data)) {
return node;
}
@@ -972,8 +972,8 @@ TExprNode::TPtr ConvertFold1BySumToLength(const TExprNode::TPtr& node, TExprCont
auto arg1 = updateLambda.Tail().Child(0);
const bool isInc = updateLambda.Tail().IsCallable("Inc");
auto arg2 = isInc ? nullptr : updateLambda.Tail().Child(1);
- if (arg1->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data ||
- (arg2 && arg2->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data)) {
+ if (arg1->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data ||
+ (arg2 && arg2->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data)) {
return node;
}
@@ -1065,8 +1065,8 @@ TExprNode::TPtr ConvertFold1ByConstMinMax(const TExprNode::TPtr& node, TExprCont
const auto& updateLambda = node->Tail();
auto arg1 = updateLambda.Tail().Child(0);
auto arg2 = updateLambda.Tail().Child(1);
- if (arg1->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data ||
- arg2->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data) {
+ if (arg1->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data ||
+ arg2->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data) {
return node;
}
@@ -1280,7 +1280,7 @@ TExprNode::TPtr PropagateConstPremapIntoCombineByKey(const TExprNode& node, TExp
return ctx.ChangeChildren(node, std::move(children));
}
-TExprNode::TPtr CountAggregateRewrite(const TCoAggregate& node, TExprContext& ctx) {
+TExprNode::TPtr CountAggregateRewrite(const TCoAggregate& node, TExprContext& ctx) {
auto keyColumns = node.Keys();
auto aggregatedColumns = node.Handlers();
if (keyColumns.Size() > 0 || aggregatedColumns.Size() != 1) {
@@ -1307,7 +1307,7 @@ TExprNode::TPtr CountAggregateRewrite(const TCoAggregate& node, TExprContext& ct
auto inputItemType = traits->Head().GetTypeAnn()->Cast<TTypeExprType>()->GetType();
const bool isOptionalColumn = inputItemType->GetKind() == ETypeAnnotationKind::Optional;
- auto init = TCoLambda(traits->Child(1));
+ auto init = TCoLambda(traits->Child(1));
bool onlyColumn = true;
bool onlyZero = true;
TExprNode::TPtr initVal;
@@ -1326,7 +1326,7 @@ TExprNode::TPtr CountAggregateRewrite(const TCoAggregate& node, TExprContext& ct
return node.Ptr();
}
- auto update = TCoLambda(traits->Child(2));
+ auto update = TCoLambda(traits->Child(2));
auto inc = update.Body().Ptr();
if (inc->IsCallable("Inc") && inc->Child(0) == update.Args().Arg(1).Raw()) {
onlyZero = false;
@@ -1340,17 +1340,17 @@ TExprNode::TPtr CountAggregateRewrite(const TCoAggregate& node, TExprContext& ct
return node.Ptr();
}
- auto save = TCoLambda(traits->Child(3));
+ auto save = TCoLambda(traits->Child(3));
if (save.Body().Raw() != save.Args().Arg(0).Raw()) {
return node.Ptr();
}
- auto load = TCoLambda(traits->Child(4));
+ auto load = TCoLambda(traits->Child(4));
if (load.Body().Raw() != load.Args().Arg(0).Raw()) {
return node.Ptr();
}
- auto merge = TCoLambda(traits->Child(5));
+ auto merge = TCoLambda(traits->Child(5));
{
auto& plus = merge.Body().Ref();
if (!plus.IsCallable("+")) {
@@ -1363,7 +1363,7 @@ TExprNode::TPtr CountAggregateRewrite(const TCoAggregate& node, TExprContext& ct
}
}
- auto finish = TCoLambda(traits->Child(6));
+ auto finish = TCoLambda(traits->Child(6));
if (finish.Body().Raw() != finish.Args().Arg(0).Raw()) {
return node.Ptr();
}
@@ -1599,7 +1599,7 @@ TExprNode::TPtr OptimizeLookup(const TExprNode::TPtr& node, TExprContext& ctx, T
}
constexpr std::initializer_list<std::string_view> FlowPriority = {
- "AssumeSorted", "AssumeUnique",
+ "AssumeSorted", "AssumeUnique",
"Map", "OrderedMap",
"Filter", "OrderedFilter",
"FlatMap", "OrderedFlatMap",
@@ -1766,9 +1766,9 @@ TExprNode::TPtr OptimizeFlatMap(const TExprNode::TPtr& node, TExprContext& ctx,
void RegisterCoFlowCallables1(TCallableOptimizerMap& map) {
using namespace std::placeholders;
-
+
map["ToFlow"] = std::bind(&OptimizeToFlow, _1, _2, _3);
-
+
map["FlatMap"] = std::bind(&OptimizeFlatMap<false>, _1, _2, _3);
map["OrderedFlatMap"] = std::bind(&OptimizeFlatMap<true>, _1, _2, _3);
@@ -2013,8 +2013,8 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) {
return node;
}
- TCoCombineByKey self(node);
- if (self.Input().Maybe<TCoFlatMapBase>()) {
+ TCoCombineByKey self(node);
+ if (self.Input().Maybe<TCoFlatMapBase>()) {
YQL_CLOG(DEBUG, Core) << "FuseCombineByKeyOverFlatmap";
return FuseCombineByKeyOverFlatmap(self, ctx);
}
@@ -2034,9 +2034,9 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) {
YQL_CLOG(DEBUG, Core) << node->Content() << " premap ToList elimination";
TExprNode::TPtr newPreMapLambda = ctx.DeepCopyLambda(preMap.Ref());
newPreMapLambda = ctx.ChangeChild(*newPreMapLambda, 1, newPreMapLambda->Child(1)->Child(0));
- return Build<TCoCombineByKey>(ctx, node->Pos())
+ return Build<TCoCombineByKey>(ctx, node->Pos())
.Input(self.Input())
- .PreMapLambda(TCoLambda(newPreMapLambda))
+ .PreMapLambda(TCoLambda(newPreMapLambda))
.KeySelectorLambda(self.KeySelectorLambda())
.InitHandlerLambda(self.InitHandlerLambda())
.UpdateHandlerLambda(self.UpdateHandlerLambda())
@@ -2052,7 +2052,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) {
return node;
}
- TCoAggregate self(node);
+ TCoAggregate self(node);
auto ret = CountAggregateRewrite(self, ctx);
if (ret != node) {
YQL_CLOG(DEBUG, Core) << "CountAggregateRewrite";
@@ -2064,230 +2064,230 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) {
map["Reverse"] = std::bind(&OptimizeReverse, _1, _2, _3);
- map["Visit"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ map["Visit"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
if (!optCtx.IsSingleUsage(node->Head())) {
- return node;
- }
-
+ return node;
+ }
+
if (!TCoVisit::Match(&node->Head())) {
- return node;
- }
-
- // Outer variant index to inner index + lambda
- THashMap<TStringBuf, std::pair<TStringBuf, TExprNode::TPtr>> innerLambdas;
- TExprNode::TPtr defValue;
- TStringBuf defOutIndex;
- TSet<TString> defInnerIndicies;
-
+ return node;
+ }
+
+ // Outer variant index to inner index + lambda
+ THashMap<TStringBuf, std::pair<TStringBuf, TExprNode::TPtr>> innerLambdas;
+ TExprNode::TPtr defValue;
+ TStringBuf defOutIndex;
+ TSet<TString> defInnerIndicies;
+
const auto& innerVisit = node->Head();
-
+
if (innerVisit.ChildrenSize() % 2 == 0) {
- // Has default value
+ // Has default value
auto innerVarType = innerVisit.GetTypeAnn()->Cast<TVariantExprType>();
- if (innerVarType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Struct) {
- for (auto item: innerVarType->GetUnderlyingType()->Cast<TStructExprType>()->GetItems()) {
- defInnerIndicies.emplace(item->GetName());
- }
- }
- else {
- for (size_t i = 0; i < innerVarType->GetUnderlyingType()->Cast<TTupleExprType>()->GetSize(); ++i) {
- defInnerIndicies.emplace(ToString(i));
- }
- }
- }
-
+ if (innerVarType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Struct) {
+ for (auto item: innerVarType->GetUnderlyingType()->Cast<TStructExprType>()->GetItems()) {
+ defInnerIndicies.emplace(item->GetName());
+ }
+ }
+ else {
+ for (size_t i = 0; i < innerVarType->GetUnderlyingType()->Cast<TTupleExprType>()->GetSize(); ++i) {
+ defInnerIndicies.emplace(ToString(i));
+ }
+ }
+ }
+
for (ui32 index = 1; index < innerVisit.ChildrenSize(); ++index) {
if (innerVisit.Child(index)->IsAtom()) {
const auto itemIndex = innerVisit.Child(index)->Content();
defInnerIndicies.erase(TString(itemIndex));
- ++index;
+ ++index;
auto lambda = innerVisit.ChildPtr(index);
- if (auto var = TMaybeNode<TCoVariant>(lambda->Child(1))) {
- innerLambdas[var.Cast().Index().Value()] = std::make_pair(itemIndex, std::move(lambda));
- }
- else {
- return node;
- }
- }
- else {
+ if (auto var = TMaybeNode<TCoVariant>(lambda->Child(1))) {
+ innerLambdas[var.Cast().Index().Value()] = std::make_pair(itemIndex, std::move(lambda));
+ }
+ else {
+ return node;
+ }
+ }
+ else {
if (auto var = TMaybeNode<TCoVariant>(innerVisit.Child(index))) {
- defOutIndex = var.Cast().Index().Value();
- defValue = var.Cast().Item().Ptr();
- }
- else {
- return node;
- }
- }
- }
-
+ defOutIndex = var.Cast().Index().Value();
+ defValue = var.Cast().Item().Ptr();
+ }
+ else {
+ return node;
+ }
+ }
+ }
+
if (innerLambdas.contains(defOutIndex)) {
- return node;
- }
-
- for (ui32 index = 1; index < node->ChildrenSize(); ++index) {
- if (node->Child(index)->IsAtom()) {
- const auto itemIndex = node->Child(index)->Content();
+ return node;
+ }
+
+ for (ui32 index = 1; index < node->ChildrenSize(); ++index) {
+ if (node->Child(index)->IsAtom()) {
+ const auto itemIndex = node->Child(index)->Content();
if (!innerLambdas.contains(itemIndex) && defOutIndex != itemIndex) {
- return node;
- }
- }
- }
-
+ return node;
+ }
+ }
+ }
+
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
- return ctx.Builder(node->Pos())
- .Callable("Visit")
+ return ctx.Builder(node->Pos())
+ .Callable("Visit")
.Add(0, innerVisit.HeadPtr())
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- ui32 index = 0;
- for (ui32 i = 1; i < node->ChildrenSize(); ++i) {
- if (node->Child(i)->IsAtom()) {
- const auto itemIndex = node->Child(i)->Content();
- auto lambda = node->ChildPtr(i + 1);
- if (auto p = innerLambdas.FindPtr(itemIndex)) {
- lambda = ctx.Builder(lambda->Pos())
- .Lambda()
- .Param("item")
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ ui32 index = 0;
+ for (ui32 i = 1; i < node->ChildrenSize(); ++i) {
+ if (node->Child(i)->IsAtom()) {
+ const auto itemIndex = node->Child(i)->Content();
+ auto lambda = node->ChildPtr(i + 1);
+ if (auto p = innerLambdas.FindPtr(itemIndex)) {
+ lambda = ctx.Builder(lambda->Pos())
+ .Lambda()
+ .Param("item")
.Apply(*lambda)
- .With(0)
+ .With(0)
.ApplyPartial(p->second->HeadPtr(), p->second->Child(1)->ChildPtr(TCoVariant::idx_Item))
- .With(0, "item")
- .Seal()
- .Done()
- .Seal()
- .Seal()
- .Build();
-
- parent.Atom(++index, p->first);
- parent.Add(++index, lambda);
- }
- else {
- lambda = ctx.Builder(lambda->Pos())
- .Lambda()
- .Param("item")
+ .With(0, "item")
+ .Seal()
+ .Done()
+ .Seal()
+ .Seal()
+ .Build();
+
+ parent.Atom(++index, p->first);
+ parent.Add(++index, lambda);
+ }
+ else {
+ lambda = ctx.Builder(lambda->Pos())
+ .Lambda()
+ .Param("item")
.Apply(*lambda)
- .With(0, defValue)
- .Seal()
- .Seal()
- .Build();
- for (auto& newItemIndex: defInnerIndicies) {
- parent.Atom(++index, newItemIndex);
- parent.Add(++index, lambda);
- }
- }
- ++i;
- }
- else {
- parent.Add(++index, node->ChildPtr(i));
- }
- }
- return parent;
- })
- .Seal()
- .Build();
- };
-
- map["VariantItem"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ .With(0, defValue)
+ .Seal()
+ .Seal()
+ .Build();
+ for (auto& newItemIndex: defInnerIndicies) {
+ parent.Atom(++index, newItemIndex);
+ parent.Add(++index, lambda);
+ }
+ }
+ ++i;
+ }
+ else {
+ parent.Add(++index, node->ChildPtr(i));
+ }
+ }
+ return parent;
+ })
+ .Seal()
+ .Build();
+ };
+
+ map["VariantItem"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
if (!optCtx.IsSingleUsage(node->Head())) {
- return node;
- }
-
+ return node;
+ }
+
if (!TCoVisit::Match(&node->Head())) {
- return node;
- }
-
+ return node;
+ }
+
const auto& visit = node->Head();
for (ui32 index = 1; index < visit.ChildrenSize(); ++index) {
if (visit.Child(index)->IsAtom()) {
- ++index;
+ ++index;
if (!TCoVariant::Match(visit.Child(index)->Child(1))) {
- return node;
- }
- }
- else {
+ return node;
+ }
+ }
+ else {
if (!TCoVariant::Match(visit.Child(index))) {
- return node;
- }
- }
- }
-
+ return node;
+ }
+ }
+ }
+
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
return ctx.Builder(visit.Pos())
- .Callable("Visit")
+ .Callable("Visit")
.Add(0, visit.HeadPtr())
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
for (ui32 i = 1; i < visit.ChildrenSize(); ++i) {
if (visit.Child(i)->IsAtom()) {
parent.Add(i, visit.ChildPtr(i));
auto visitLambda = visit.Child(i + 1);
- parent.Lambda(i + 1, visitLambda->Pos())
- .Param("item")
+ parent.Lambda(i + 1, visitLambda->Pos())
+ .Param("item")
.ApplyPartial(visitLambda->HeadPtr(), visitLambda->Child(1)->ChildPtr(TCoVariant::idx_Item))
- .With(0, "item")
- .Seal()
- .Seal();
- ++i;
- }
- else {
+ .With(0, "item")
+ .Seal()
+ .Seal();
+ ++i;
+ }
+ else {
parent.Add(i, visit.Child(i)->ChildPtr(TCoVariant::idx_Item));
- }
- }
- return parent;
- })
- .Seal()
- .Build();
- };
-
- map["SkipNullMembers"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ }
+ }
+ return parent;
+ })
+ .Seal()
+ .Build();
+ };
+
+ map["SkipNullMembers"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
if (!optCtx.IsSingleUsage(node->Head())) {
- return node;
- }
-
- const auto skipNullMembers = TCoSkipNullMembers(node);
- if (!skipNullMembers.Members()) {
- return node;
- }
-
- if (auto maybeFlatmap = skipNullMembers.Input().Maybe<TCoFlatMapBase>()) {
- auto flatmap = maybeFlatmap.Cast();
-
- TMaybe<THashSet<TStringBuf>> passthroughFields;
+ return node;
+ }
+
+ const auto skipNullMembers = TCoSkipNullMembers(node);
+ if (!skipNullMembers.Members()) {
+ return node;
+ }
+
+ if (auto maybeFlatmap = skipNullMembers.Input().Maybe<TCoFlatMapBase>()) {
+ auto flatmap = maybeFlatmap.Cast();
+
+ TMaybe<THashSet<TStringBuf>> passthroughFields;
if (IsPassthroughFlatMap(flatmap, &passthroughFields)
&& !IsTablePropsDependent(flatmap.Lambda().Body().Ref())
// SkipNullMembers doesn't support optional items
&& flatmap.Lambda().Args().Arg(0).Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional)
{
- bool hasAllMembers = true;
- if (passthroughFields) {
- for (const auto& member : skipNullMembers.Members().Cast()) {
- if (!passthroughFields->contains(member)) {
- hasAllMembers = false;
- break;
- }
- }
- }
-
- if (hasAllMembers) {
- YQL_CLOG(DEBUG, Core) << node->Content() << "OverFlatmap";
- return ctx.Builder(flatmap.Pos())
- .Callable(flatmap.CallableName())
- .Callable(0, TCoSkipNullMembers::CallableName())
- .Add(0, flatmap.Input().Ptr())
- .Add(1, skipNullMembers.Members().Cast().Ptr())
- .Seal()
- .Add(1, flatmap.Lambda().Ptr())
- .Seal()
- .Build();
- }
- }
- }
-
+ bool hasAllMembers = true;
+ if (passthroughFields) {
+ for (const auto& member : skipNullMembers.Members().Cast()) {
+ if (!passthroughFields->contains(member)) {
+ hasAllMembers = false;
+ break;
+ }
+ }
+ }
+
+ if (hasAllMembers) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << "OverFlatmap";
+ return ctx.Builder(flatmap.Pos())
+ .Callable(flatmap.CallableName())
+ .Callable(0, TCoSkipNullMembers::CallableName())
+ .Add(0, flatmap.Input().Ptr())
+ .Add(1, skipNullMembers.Members().Cast().Ptr())
+ .Seal()
+ .Add(1, flatmap.Lambda().Ptr())
+ .Seal()
+ .Build();
+ }
+ }
+ }
+
if (node->Head().IsCallable({"ForwardList", "FromFlow"})) {
YQL_CLOG(DEBUG, Core) << "Swap " << node->Content() << " with " << node->Head().Content();
return ctx.SwapWithHead(*node);
}
- return node;
- };
+ return node;
+ };
map["Exists"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
if (!optCtx.IsSingleUsage(node->Head())) {
@@ -2296,7 +2296,7 @@ void RegisterCoFlowCallables1(TCallableOptimizerMap& map) {
if (node->Head().IsCallable("Lookup")) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
- return ctx.RenameNode(node->Head(), "Contains");
+ return ctx.RenameNode(node->Head(), "Contains");
}
return node;
diff --git a/ydb/library/yql/core/common_opt/yql_co_flow2.cpp b/ydb/library/yql/core/common_opt/yql_co_flow2.cpp
index 88f9140031..6b0240b693 100644
--- a/ydb/library/yql/core/common_opt/yql_co_flow2.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_flow2.cpp
@@ -1,4 +1,4 @@
-#include "yql_co_extr_members.h"
+#include "yql_co_extr_members.h"
#include "yql_co.h"
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
@@ -16,7 +16,7 @@ namespace {
using namespace NNodes;
TExprNode::TPtr AggregateSubsetFieldsAnalyzer(const TCoAggregate& node, TExprContext& ctx, const TParentsMap& parentsMap) {
- auto inputType = node.Input().Ref().GetTypeAnn();
+ auto inputType = node.Input().Ref().GetTypeAnn();
auto structType = inputType->GetKind() == ETypeAnnotationKind::List
? inputType->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>()
: inputType->Cast<TStreamExprType>()->GetItemType()->Cast<TStructExprType>();
@@ -50,7 +50,7 @@ TExprNode::TPtr AggregateSubsetFieldsAnalyzer(const TCoAggregate& node, TExprCon
}
else {
auto traits = x.Ref().Child(1);
- auto structType = traits->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
+ auto structType = traits->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
for (const auto& item : structType->GetItems()) {
usedFields.insert(item->GetName());
}
@@ -147,7 +147,7 @@ TExprNode::TPtr FuseAndTerms(TPositionHandle position, const TExprNode::TListTyp
return prevAndNode;
}
-TExprNode::TPtr ConstantPredicatePushdownOverEquiJoin(TExprNode::TPtr equiJoin, TExprNode::TPtr predicate, bool ordered, TExprContext& ctx) {
+TExprNode::TPtr ConstantPredicatePushdownOverEquiJoin(TExprNode::TPtr equiJoin, TExprNode::TPtr predicate, bool ordered, TExprContext& ctx) {
auto lambda = ctx.Builder(predicate->Pos())
.Lambda()
.Param("row")
@@ -160,7 +160,7 @@ TExprNode::TPtr ConstantPredicatePushdownOverEquiJoin(TExprNode::TPtr equiJoin,
for (ui32 i = 0; i < inputsCount; ++i) {
ret->ChildRef(i) = ctx.ShallowCopy(*ret->Child(i));
ret->Child(i)->ChildRef(0) = ctx.Builder(predicate->Pos())
- .Callable(ordered ? "OrderedFilter" : "Filter")
+ .Callable(ordered ? "OrderedFilter" : "Filter")
.Add(0, ret->Child(i)->ChildPtr(0))
.Add(1, lambda)
.Seal()
@@ -254,7 +254,7 @@ void GatherOptionalKeyColumns(TExprNode::TPtr joinTree, const TJoinLabels& label
TExprNode::TPtr SingleInputPredicatePushdownOverEquiJoin(TExprNode::TPtr equiJoin, TExprNode::TPtr predicate,
const TSet<TStringBuf>& usedFields, TExprNode::TPtr args, const TJoinLabels& labels,
- ui32 firstCandidate, const TMap<TStringBuf, TVector<TStringBuf>>& renameMap, bool ordered, TExprContext& ctx) {
+ ui32 firstCandidate, const TMap<TStringBuf, TVector<TStringBuf>>& renameMap, bool ordered, TExprContext& ctx) {
auto inputsCount = equiJoin->ChildrenSize() - 2;
auto joinTree = equiJoin->Child(inputsCount);
TMap<TString, TSet<TString>> aliases;
@@ -320,7 +320,7 @@ TExprNode::TPtr SingleInputPredicatePushdownOverEquiJoin(TExprNode::TPtr equiJoi
// then apply predicate
newInput = ctx.Builder(predicate->Pos())
- .Callable(ordered ? "OrderedFilter" : "Filter")
+ .Callable(ordered ? "OrderedFilter" : "Filter")
.Add(0, newInput)
.Lambda(1)
.Param("row")
@@ -383,9 +383,9 @@ TExprNode::TPtr SingleInputPredicatePushdownOverEquiJoin(TExprNode::TPtr equiJoi
return ret;
}
-TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ctx, const TParentsMap& parentsMap) {
+TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ctx, const TParentsMap& parentsMap) {
auto equiJoin = node.Input();
- auto structType = equiJoin.Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType()
+ auto structType = equiJoin.Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType()
->Cast<TStructExprType>();
if (structType->GetSize() == 0) {
return node.Ptr();
@@ -393,11 +393,11 @@ TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ct
TExprNode::TPtr structNode;
if (IsRenameFlatMap(node, structNode)) {
- YQL_CLOG(DEBUG, Core) << "Rename in " << node.CallableName() << " over EquiJoin";
+ YQL_CLOG(DEBUG, Core) << "Rename in " << node.CallableName() << " over EquiJoin";
auto joinSettings = equiJoin.Ref().ChildPtr(equiJoin.Ref().ChildrenSize() - 1);
auto renameMap = LoadJoinRenameMap(*joinSettings);
joinSettings = RemoveSetting(*joinSettings, "rename", ctx);
- auto structType = equiJoin.Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType()
+ auto structType = equiJoin.Ref().GetTypeAnn()->Cast<TListExprType>()->GetItemType()
->Cast<TStructExprType>();
THashSet<TStringBuf> usedFields;
TMap<TStringBuf, TVector<TStringBuf>> memberUsageMap;
@@ -456,7 +456,7 @@ TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ct
auto& arg = node.Lambda().Args().Arg(0).Ref();
auto body = node.Lambda().Body().Ptr();
if (HaveFieldsSubset(body, arg, usedFields, parentsMap)) {
- YQL_CLOG(DEBUG, Core) << "FieldsSubset in " << node.CallableName() << " over EquiJoin";
+ YQL_CLOG(DEBUG, Core) << "FieldsSubset in " << node.CallableName() << " over EquiJoin";
auto joinSettings = equiJoin.Ref().ChildPtr(equiJoin.Ref().ChildrenSize() - 1);
auto renameMap = LoadJoinRenameMap(*joinSettings);
joinSettings = RemoveSetting(*joinSettings, "rename", ctx);
@@ -475,7 +475,7 @@ TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ct
updatedEquiJoin->ChildRef(updatedEquiJoin->ChildrenSize() - 1) = joinSettings;
return ctx.Builder(node.Pos())
- .Callable(node.CallableName())
+ .Callable(node.CallableName())
.Add(0, updatedEquiJoin)
.Add(1, newLambda)
.Seal()
@@ -490,7 +490,7 @@ TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ct
TJoinLabels labels;
for (ui32 i = 0; i < equiJoin.Ref().ChildrenSize() - 2; ++i) {
auto err = labels.Add(ctx, *equiJoin.Ref().Child(i)->Child(1),
- equiJoin.Ref().Child(i)->Child(0)->GetTypeAnn()->Cast<TListExprType>()
+ equiJoin.Ref().Child(i)->Child(0)->GetTypeAnn()->Cast<TListExprType>()
->GetItemType()->Cast<TStructExprType>());
if (err) {
ctx.AddError(*err);
@@ -513,8 +513,8 @@ TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ct
}
}
- const bool ordered = node.Maybe<TCoOrderedFlatMap>().IsValid();
-
+ const bool ordered = node.Maybe<TCoOrderedFlatMap>().IsValid();
+
for (auto& andTerm : andTerms) {
if (andTerm->IsCallable("Likely")) {
continue;
@@ -548,14 +548,14 @@ TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ct
if (inputs.size() == 0) {
YQL_CLOG(DEBUG, Core) << "ConstantPredicatePushdownOverEquiJoin";
- ret = ConstantPredicatePushdownOverEquiJoin(equiJoin.Ptr(), andTerm, ordered, ctx);
+ ret = ConstantPredicatePushdownOverEquiJoin(equiJoin.Ptr(), andTerm, ordered, ctx);
extraPredicate = FuseAndTerms(node.Pos(), andTerms, andTerm, ctx);
break;
}
if (inputs.size() == 1) {
auto newJoin = SingleInputPredicatePushdownOverEquiJoin(equiJoin.Ptr(), andTerm, usedFields,
- node.Lambda().Args().Ptr(), labels, *inputs.begin(), renameMap, ordered, ctx);
+ node.Lambda().Args().Ptr(), labels, *inputs.begin(), renameMap, ordered, ctx);
if (newJoin != equiJoin.Ptr()) {
YQL_CLOG(DEBUG, Core) << "SingleInputPredicatePushdownOverEquiJoin";
ret = newJoin;
@@ -571,7 +571,7 @@ TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ct
if (extraPredicate) {
ret = ctx.Builder(node.Pos())
- .Callable(ordered ? "OrderedFilter" : "Filter")
+ .Callable(ordered ? "OrderedFilter" : "Filter")
.Add(0, std::move(ret))
.Lambda(1)
.Param("item")
@@ -582,12 +582,12 @@ TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ct
}
if (value != &row) {
- TString name = node.Lambda().Body().Ref().Content().StartsWith("Flat") ? "FlatMap" : "Map";
- if (ordered) {
- name.prepend("Ordered");
- }
+ TString name = node.Lambda().Body().Ref().Content().StartsWith("Flat") ? "FlatMap" : "Map";
+ if (ordered) {
+ name.prepend("Ordered");
+ }
ret = ctx.Builder(node.Pos())
- .Callable(name)
+ .Callable(name)
.Add(0, std::move(ret))
.Lambda(1)
.Param("item")
@@ -603,7 +603,7 @@ TExprNode::TPtr FlatMapOverEquiJoin(const TCoFlatMapBase& node, TExprContext& ct
return node.Ptr();
}
-TExprNode::TPtr FlatMapSubsetFields(const TCoFlatMapBase& node, TExprContext& ctx, const TParentsMap& parentsMap) {
+TExprNode::TPtr FlatMapSubsetFields(const TCoFlatMapBase& node, TExprContext& ctx, const TParentsMap& parentsMap) {
auto it = parentsMap.find(node.Input().Raw());
YQL_ENSURE(it != parentsMap.cend());
auto inputParentsCount = it->second.size();
@@ -613,7 +613,7 @@ TExprNode::TPtr FlatMapSubsetFields(const TCoFlatMapBase& node, TExprContext& ct
}
auto itemArg = node.Lambda().Args().Arg(0);
- auto itemType = itemArg.Ref().GetTypeAnn();
+ auto itemType = itemArg.Ref().GetTypeAnn();
if (itemType->GetKind() != ETypeAnnotationKind::Struct) {
return node.Ptr();
}
@@ -635,23 +635,23 @@ TExprNode::TPtr FlatMapSubsetFields(const TCoFlatMapBase& node, TExprContext& ct
}
}
- return Build<TCoFlatMapBase>(ctx, node.Pos())
- .CallableName(node.Ref().Content())
- .Input<TCoExtractMembers>()
- .Input(node.Input())
- .Members()
- .Add(fieldNodes)
+ return Build<TCoFlatMapBase>(ctx, node.Pos())
+ .CallableName(node.Ref().Content())
+ .Input<TCoExtractMembers>()
+ .Input(node.Input())
+ .Members()
+ .Add(fieldNodes)
.Build()
- .Build()
- .Lambda()
- .Args({"item"})
- .Body<TExprApplier>()
- .Apply(node.Lambda())
- .With(0, "item")
+ .Build()
+ .Lambda()
+ .Args({"item"})
+ .Body<TExprApplier>()
+ .Apply(node.Lambda())
+ .With(0, "item")
.Build()
- .Build()
- .Done()
- .Ptr();
+ .Build()
+ .Done()
+ .Ptr();
}
TExprNode::TPtr RenameJoinTable(TPositionHandle pos, TExprNode::TPtr table,
@@ -886,7 +886,7 @@ TExprNode::TPtr FuseEquiJoins(const TExprNode::TPtr& node, ui32 upstreamIndex, T
}
// fill remaining upstream columns
- for (const auto& item : upstreamList->GetTypeAnn()->Cast<TListExprType>()
+ for (const auto& item : upstreamList->GetTypeAnn()->Cast<TListExprType>()
->GetItemType()->Cast<TStructExprType>()->GetItems()) {
auto columnName = TString(item->GetName());
if (upstreamColumnsBackRename.count(columnName)) {
@@ -1197,7 +1197,7 @@ const TTypeAnnotationNode* GetCanaryOutputType(const TStructExprType& outputType
}
TExprNode::TPtr BuildOutputFlattenMembersArg(const TCoEquiJoinInput& input, const TExprNode::TPtr& inputArg,
- const TString& canaryName, const TStructExprType& canaryResultTypeWithoutRenames, bool keepSys, TExprContext& ctx)
+ const TString& canaryName, const TStructExprType& canaryResultTypeWithoutRenames, bool keepSys, TExprContext& ctx)
{
YQL_ENSURE(input.Scope().Ref().IsAtom());
TStringBuf label = input.Scope().Ref().Content();
@@ -1245,9 +1245,9 @@ TExprNode::TPtr BuildOutputFlattenMembersArg(const TCoEquiJoinInput& input, cons
TExprNode::TListType membersForCheck;
auto flatMapInputItems = flatMapInputItem->Cast<TStructExprType>()->GetItems();
- if (!keepSys) {
- EraseIf(flatMapInputItems, [](const TItemExprType* item) { return item->GetName().StartsWith("_yql_sys_"); });
- }
+ if (!keepSys) {
+ EraseIf(flatMapInputItems, [](const TItemExprType* item) { return item->GetName().StartsWith("_yql_sys_"); });
+ }
flatMapInputItems.push_back(ctx.MakeType<TItemExprType>(canaryName, ctx.MakeType<TDataExprType>(EDataSlot::Bool)));
for (auto& item : flatMapInputItems) {
if (item->GetItemType()->GetKind() != ETypeAnnotationKind::Optional) {
@@ -1262,10 +1262,10 @@ TExprNode::TPtr BuildOutputFlattenMembersArg(const TCoEquiJoinInput& input, cons
.Atom(0, labelPrefix)
.Callable(1, "IfPresent")
.Callable(0, "FilterNullMembers")
- .Callable(0, "AssumeAllMembersNullableAtOnce")
- .Callable(0, "Just")
- .Add(0, std::move(myStruct))
- .Seal()
+ .Callable(0, "AssumeAllMembersNullableAtOnce")
+ .Callable(0, "Just")
+ .Add(0, std::move(myStruct))
+ .Seal()
.Seal()
.Add(1, std::move(checkedMembersList))
.Seal()
@@ -1313,15 +1313,15 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont
return node;
}
- bool keepSys = false;
+ bool keepSys = false;
auto settings = node->ChildPtr(inputsCount + 1);
for (auto& child : settings->Children()) {
if (child->Child(0)->Content() == "flatten") {
return node;
}
- if (child->Child(0)->Content() == "keep_sys") {
- keepSys = true;
- }
+ if (child->Child(0)->Content() == "keep_sys") {
+ keepSys = true;
+ }
}
static const TStringBuf canaryBaseName = "_yql_canary_";
@@ -1360,9 +1360,9 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont
auto flatMapInputItem = GetSequenceItemType(flatMap.Input(), false);
auto structItems = flatMapInputItem->Cast<TStructExprType>()->GetItems();
- if (!keepSys) {
- EraseIf(structItems, [](const TItemExprType* item) { return item->GetName().StartsWith("_yql_sys_"); });
- }
+ if (!keepSys) {
+ EraseIf(structItems, [](const TItemExprType* item) { return item->GetName().StartsWith("_yql_sys_"); });
+ }
TString canaryName = TStringBuilder() << canaryBaseName << i;
structItems.push_back(ctx.MakeType<TItemExprType>(canaryName, ctx.MakeType<TDataExprType>(EDataSlot::Bool)));
@@ -1391,10 +1391,10 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont
auto status = ValidateEquiJoinOptions(node->Pos(), *settingsWithoutRenames, options, ctx);
YQL_ENSURE(status == IGraphTransformer::TStatus::Ok);
- status = EquiJoinAnnotation(node->Pos(), canaryResultType, canaryLabels,
+ status = EquiJoinAnnotation(node->Pos(), canaryResultType, canaryLabels,
*joinTreeWithInputRenames, options, ctx);
- YQL_ENSURE(status == IGraphTransformer::TStatus::Ok);
-
+ YQL_ENSURE(status == IGraphTransformer::TStatus::Ok);
+
status = EquiJoinAnnotation(node->Pos(), noRenamesResultType, actualLabels,
*joinTree, options, ctx);
YQL_ENSURE(status == IGraphTransformer::TStatus::Ok);
@@ -1461,7 +1461,7 @@ TExprNode::TPtr PullUpFlatMapOverEquiJoin(const TExprNode::TPtr& node, TExprCont
);
}
- auto flattenMembersArg = BuildOutputFlattenMembersArg(input, afterJoinArg, canaryName, *canaryResultType, keepSys, ctx);
+ auto flattenMembersArg = BuildOutputFlattenMembersArg(input, afterJoinArg, canaryName, *canaryResultType, keepSys, ctx);
if (flattenMembersArg) {
flattenMembersArgs.push_back(flattenMembersArg);
}
@@ -1546,8 +1546,8 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
map["FromFlow"] = std::bind(&OptimizeFromFlow, _1, _2, _3);
map["Collect"] = std::bind(&OptimizeCollect, _1, _2, _3);
- map["FlatMap"] = map["OrderedFlatMap"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- TCoFlatMapBase self(node);
+ map["FlatMap"] = map["OrderedFlatMap"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ TCoFlatMapBase self(node);
if (!optCtx.IsSingleUsage(self.Input().Ref())) {
return node;
}
@@ -1555,97 +1555,97 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
if (self.Input().Ref().IsCallable("EquiJoin")) {
auto ret = FlatMapOverEquiJoin(self, ctx, *optCtx.ParentsMap);
if (ret != node) {
- YQL_CLOG(DEBUG, Core) << node->Content() << "OverEquiJoin";
+ YQL_CLOG(DEBUG, Core) << node->Content() << "OverEquiJoin";
return ret;
}
}
- if (self.Input().Ref().IsCallable(TCoGroupingCore::CallableName())) {
- auto groupingCore = self.Input().Cast<TCoGroupingCore>();
- const TExprNode* extract = nullptr;
- // Find pattern: (FlatMap (GroupingCore ...) (lambda (x) ( ... (ExtractMembers (Nth x '1) ...))))
- const auto arg = self.Lambda().Args().Arg(0).Raw();
- if (const auto parents = optCtx.ParentsMap->find(arg); parents != optCtx.ParentsMap->cend()) {
- for (const auto& parent : parents->second) {
- if (parent->IsCallable(TCoNth::CallableName()) && &parent->Head() == arg && parent->Tail().Content() == "1") {
- if (const auto nthParents = optCtx.ParentsMap->find(parent); nthParents != optCtx.ParentsMap->cend()) {
- if (nthParents->second.size() == 1 && (*nthParents->second.begin())->IsCallable(TCoExtractMembers::CallableName())) {
- extract = *nthParents->second.begin();
- break;
- }
- }
- }
- }
- }
- if (extract) {
- if (const auto handler = groupingCore.ConvertHandler()) {
- auto newBody = Build<TCoCastStruct>(ctx, handler.Cast().Body().Pos())
- .Struct(handler.Cast().Body())
- .Type(ExpandType(handler.Cast().Body().Pos(), *GetSeqItemType(extract->GetTypeAnn()), ctx))
- .Done();
-
- groupingCore = Build<TCoGroupingCore>(ctx, groupingCore.Pos())
- .InitFrom(groupingCore)
- .ConvertHandler()
- .Args({"item"})
- .Body<TExprApplier>()
- .Apply(newBody)
- .With(handler.Cast().Args().Arg(0), "item")
- .Build()
- .Build()
- .Done();
-
- YQL_CLOG(DEBUG, Core) << "Pull out " << extract->Content() << " from " << node->Content() << " to " << groupingCore.Ref().Content() << " handler";
- return Build<TCoFlatMapBase>(ctx, node->Pos())
- .CallableName(node->Content())
- .Input(groupingCore)
- .Lambda(ctx.DeepCopyLambda(self.Lambda().Ref()))
- .Done().Ptr();
- }
-
- std::map<std::string_view, TExprNode::TPtr> usedFields;
- auto fields = extract->Tail().ChildrenList();
- std::for_each(fields.cbegin(), fields.cend(), [&](const TExprNode::TPtr& field) { usedFields.emplace(field->Content(), field); });
-
- if (HaveFieldsSubset(groupingCore.KeyExtractor().Body().Ptr(), groupingCore.KeyExtractor().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
- && !usedFields.empty()
- && HaveFieldsSubset(groupingCore.GroupSwitch().Body().Ptr(), groupingCore.GroupSwitch().Args().Arg(1).Ref(), usedFields, *optCtx.ParentsMap, false)
- && !usedFields.empty()
- && usedFields.size() < GetSeqItemType(groupingCore.Input().Ref().GetTypeAnn())->Cast<TStructExprType>()->GetSize()) {
- if (usedFields.size() != fields.size()) {
- fields.reserve(usedFields.size());
- fields.clear();
- std::transform(usedFields.begin(), usedFields.end(), std::back_inserter(fields),
- [](std::pair<const std::string_view, TExprNode::TPtr>& item){ return std::move(item.second); });
- }
-
- YQL_CLOG(DEBUG, Core) << "Pull out " << extract->Content() << " from " << node->Content() << " to " << groupingCore.Ref().Content() << " input";
- return Build<TCoFlatMapBase>(ctx, node->Pos())
- .CallableName(node->Content())
- .Input<TCoGroupingCore>()
- .Input<TCoExtractMembers>()
- .Input(groupingCore.Input())
- .Members()
- .Add(std::move(fields))
- .Build()
- .Build()
- .GroupSwitch(ctx.DeepCopyLambda(groupingCore.GroupSwitch().Ref()))
- .KeyExtractor(ctx.DeepCopyLambda(groupingCore.KeyExtractor().Ref()))
- .Build()
- .Lambda(ctx.DeepCopyLambda(self.Lambda().Ref()))
- .Done().Ptr();
- }
- }
- }
-
+ if (self.Input().Ref().IsCallable(TCoGroupingCore::CallableName())) {
+ auto groupingCore = self.Input().Cast<TCoGroupingCore>();
+ const TExprNode* extract = nullptr;
+ // Find pattern: (FlatMap (GroupingCore ...) (lambda (x) ( ... (ExtractMembers (Nth x '1) ...))))
+ const auto arg = self.Lambda().Args().Arg(0).Raw();
+ if (const auto parents = optCtx.ParentsMap->find(arg); parents != optCtx.ParentsMap->cend()) {
+ for (const auto& parent : parents->second) {
+ if (parent->IsCallable(TCoNth::CallableName()) && &parent->Head() == arg && parent->Tail().Content() == "1") {
+ if (const auto nthParents = optCtx.ParentsMap->find(parent); nthParents != optCtx.ParentsMap->cend()) {
+ if (nthParents->second.size() == 1 && (*nthParents->second.begin())->IsCallable(TCoExtractMembers::CallableName())) {
+ extract = *nthParents->second.begin();
+ break;
+ }
+ }
+ }
+ }
+ }
+ if (extract) {
+ if (const auto handler = groupingCore.ConvertHandler()) {
+ auto newBody = Build<TCoCastStruct>(ctx, handler.Cast().Body().Pos())
+ .Struct(handler.Cast().Body())
+ .Type(ExpandType(handler.Cast().Body().Pos(), *GetSeqItemType(extract->GetTypeAnn()), ctx))
+ .Done();
+
+ groupingCore = Build<TCoGroupingCore>(ctx, groupingCore.Pos())
+ .InitFrom(groupingCore)
+ .ConvertHandler()
+ .Args({"item"})
+ .Body<TExprApplier>()
+ .Apply(newBody)
+ .With(handler.Cast().Args().Arg(0), "item")
+ .Build()
+ .Build()
+ .Done();
+
+ YQL_CLOG(DEBUG, Core) << "Pull out " << extract->Content() << " from " << node->Content() << " to " << groupingCore.Ref().Content() << " handler";
+ return Build<TCoFlatMapBase>(ctx, node->Pos())
+ .CallableName(node->Content())
+ .Input(groupingCore)
+ .Lambda(ctx.DeepCopyLambda(self.Lambda().Ref()))
+ .Done().Ptr();
+ }
+
+ std::map<std::string_view, TExprNode::TPtr> usedFields;
+ auto fields = extract->Tail().ChildrenList();
+ std::for_each(fields.cbegin(), fields.cend(), [&](const TExprNode::TPtr& field) { usedFields.emplace(field->Content(), field); });
+
+ if (HaveFieldsSubset(groupingCore.KeyExtractor().Body().Ptr(), groupingCore.KeyExtractor().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
+ && !usedFields.empty()
+ && HaveFieldsSubset(groupingCore.GroupSwitch().Body().Ptr(), groupingCore.GroupSwitch().Args().Arg(1).Ref(), usedFields, *optCtx.ParentsMap, false)
+ && !usedFields.empty()
+ && usedFields.size() < GetSeqItemType(groupingCore.Input().Ref().GetTypeAnn())->Cast<TStructExprType>()->GetSize()) {
+ if (usedFields.size() != fields.size()) {
+ fields.reserve(usedFields.size());
+ fields.clear();
+ std::transform(usedFields.begin(), usedFields.end(), std::back_inserter(fields),
+ [](std::pair<const std::string_view, TExprNode::TPtr>& item){ return std::move(item.second); });
+ }
+
+ YQL_CLOG(DEBUG, Core) << "Pull out " << extract->Content() << " from " << node->Content() << " to " << groupingCore.Ref().Content() << " input";
+ return Build<TCoFlatMapBase>(ctx, node->Pos())
+ .CallableName(node->Content())
+ .Input<TCoGroupingCore>()
+ .Input<TCoExtractMembers>()
+ .Input(groupingCore.Input())
+ .Members()
+ .Add(std::move(fields))
+ .Build()
+ .Build()
+ .GroupSwitch(ctx.DeepCopyLambda(groupingCore.GroupSwitch().Ref()))
+ .KeyExtractor(ctx.DeepCopyLambda(groupingCore.KeyExtractor().Ref()))
+ .Build()
+ .Lambda(ctx.DeepCopyLambda(self.Lambda().Ref()))
+ .Done().Ptr();
+ }
+ }
+ }
+
if (self.Input().Ref().IsCallable("Take") || self.Input().Ref().IsCallable("Skip")
- || self.Input().Maybe<TCoExtendBase>()) {
+ || self.Input().Maybe<TCoExtendBase>()) {
auto& arg = self.Lambda().Args().Arg(0).Ref();
auto body = self.Lambda().Body().Ptr();
TSet<TStringBuf> usedFields;
if (HaveFieldsSubset(body, arg, usedFields, *optCtx.ParentsMap)) {
- YQL_CLOG(DEBUG, Core) << "FieldsSubset in " << node->Content() << " over " << self.Input().Ref().Content();
+ YQL_CLOG(DEBUG, Core) << "FieldsSubset in " << node->Content() << " over " << self.Input().Ref().Content();
TSet<TString> fields;
for (auto& x : usedFields) {
fields.emplace(TString(x));
@@ -1654,7 +1654,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
TExprNode::TListType filteredInputs;
for (ui32 index = 0; index < self.Input().Ref().ChildrenSize(); ++index) {
auto x = self.Input().Ref().ChildPtr(index);
- if (!self.Input().Maybe<TCoExtendBase>() && index > 0) {
+ if (!self.Input().Maybe<TCoExtendBase>() && index > 0) {
filteredInputs.push_back(x);
continue;
}
@@ -1664,7 +1664,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
auto newInput = ctx.ChangeChildren(self.Input().Ref(), std::move(filteredInputs));
return ctx.Builder(node->Pos())
- .Callable(node->Content())
+ .Callable(node->Content())
.Add(0, newInput)
.Lambda(1)
.Param("item")
@@ -1677,125 +1677,125 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
auto ret = FlatMapSubsetFields(self, ctx, *optCtx.ParentsMap);
if (ret != node) {
- YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
+ YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
return ret;
}
return node;
};
- map[TCoGroupingCore::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- TCoGroupingCore self(node);
+ map[TCoGroupingCore::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ TCoGroupingCore self(node);
+ if (!optCtx.IsSingleUsage(self.Input().Ref())) {
+ return node;
+ }
+
+ if (!self.ConvertHandler()) {
+ return node;
+ }
+
+ std::map<std::string_view, TExprNode::TPtr> usedFields;
+ if (HaveFieldsSubset(self.ConvertHandler().Cast().Body().Ptr(), self.ConvertHandler().Cast().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
+ && !usedFields.empty()
+ && HaveFieldsSubset(self.KeyExtractor().Body().Ptr(), self.KeyExtractor().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
+ && !usedFields.empty()
+ && HaveFieldsSubset(self.GroupSwitch().Body().Ptr(), self.GroupSwitch().Args().Arg(1).Ref(), usedFields, *optCtx.ParentsMap, false)
+ && !usedFields.empty()
+ && usedFields.size() < GetSeqItemType(self.Input().Ref().GetTypeAnn())->Cast<TStructExprType>()->GetSize())
+ {
+ TExprNode::TListType fields;
+ fields.reserve(usedFields.size());
+ std::transform(usedFields.begin(), usedFields.end(), std::back_inserter(fields),
+ [](std::pair<const std::string_view, TExprNode::TPtr>& item){ return std::move(item.second); });
+
+ YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
+ return Build<TCoGroupingCore>(ctx, node->Pos())
+ .Input<TCoExtractMembers>()
+ .Input(self.Input())
+ .Members()
+ .Add(std::move(fields))
+ .Build()
+ .Build()
+ .GroupSwitch(ctx.DeepCopyLambda(self.GroupSwitch().Ref()))
+ .KeyExtractor(ctx.DeepCopyLambda(self.KeyExtractor().Ref()))
+ .ConvertHandler(ctx.DeepCopyLambda(self.ConvertHandler().Ref()))
+ .Done().Ptr();
+ }
+ return node;
+ };
+
+ map["CombineByKey"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ TCoCombineByKey self(node);
if (!optCtx.IsSingleUsage(self.Input().Ref())) {
- return node;
- }
-
- if (!self.ConvertHandler()) {
- return node;
- }
-
- std::map<std::string_view, TExprNode::TPtr> usedFields;
- if (HaveFieldsSubset(self.ConvertHandler().Cast().Body().Ptr(), self.ConvertHandler().Cast().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
- && !usedFields.empty()
- && HaveFieldsSubset(self.KeyExtractor().Body().Ptr(), self.KeyExtractor().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
- && !usedFields.empty()
- && HaveFieldsSubset(self.GroupSwitch().Body().Ptr(), self.GroupSwitch().Args().Arg(1).Ref(), usedFields, *optCtx.ParentsMap, false)
- && !usedFields.empty()
- && usedFields.size() < GetSeqItemType(self.Input().Ref().GetTypeAnn())->Cast<TStructExprType>()->GetSize())
- {
- TExprNode::TListType fields;
- fields.reserve(usedFields.size());
- std::transform(usedFields.begin(), usedFields.end(), std::back_inserter(fields),
- [](std::pair<const std::string_view, TExprNode::TPtr>& item){ return std::move(item.second); });
-
- YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
- return Build<TCoGroupingCore>(ctx, node->Pos())
- .Input<TCoExtractMembers>()
- .Input(self.Input())
- .Members()
- .Add(std::move(fields))
- .Build()
- .Build()
- .GroupSwitch(ctx.DeepCopyLambda(self.GroupSwitch().Ref()))
- .KeyExtractor(ctx.DeepCopyLambda(self.KeyExtractor().Ref()))
- .ConvertHandler(ctx.DeepCopyLambda(self.ConvertHandler().Ref()))
- .Done().Ptr();
- }
- return node;
- };
-
- map["CombineByKey"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- TCoCombineByKey self(node);
- if (!optCtx.IsSingleUsage(self.Input().Ref())) {
- return node;
- }
-
- auto itemArg = self.PreMapLambda().Args().Arg(0);
- auto itemType = itemArg.Ref().GetTypeAnn();
- if (itemType->GetKind() != ETypeAnnotationKind::Struct) {
- return node;
- }
-
- auto itemStructType = itemType->Cast<TStructExprType>();
- if (itemStructType->GetSize() == 0) {
- return node;
- }
-
- TSet<TStringBuf> usedFields;
- if (!HaveFieldsSubset(self.PreMapLambda().Body().Ptr(), itemArg.Ref(), usedFields, *optCtx.ParentsMap)) {
- return node;
- }
-
- TExprNode::TPtr newInput;
- if (self.Input().Ref().IsCallable("Take") || self.Input().Ref().IsCallable("Skip") || self.Input().Maybe<TCoExtendBase>()) {
- TSet<TString> fields;
- for (auto& x : usedFields) {
- fields.emplace(TString(x));
- }
-
- TExprNode::TListType filteredInputs;
- for (ui32 index = 0; index < self.Input().Ref().ChildrenSize(); ++index) {
- auto x = self.Input().Ref().ChildPtr(index);
- if (!self.Input().Maybe<TCoExtendBase>() && index > 0) {
- filteredInputs.push_back(x);
- continue;
- }
-
- filteredInputs.push_back(FilterByFields(node->Pos(), x, fields, ctx, false));
- }
-
- YQL_CLOG(DEBUG, Core) << "FieldsSubset in " << node->Content() << " over " << self.Input().Ref().Content();
- newInput = ctx.ChangeChildren(self.Input().Ref(), std::move(filteredInputs));
- }
- else {
- TExprNode::TListType fieldNodes;
- for (auto& item : itemStructType->GetItems()) {
- if (usedFields.contains(item->GetName())) {
- fieldNodes.push_back(ctx.NewAtom(self.Pos(), item->GetName()));
- }
- }
-
- YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
- newInput = Build<TCoExtractMembers>(ctx, self.Input().Pos())
- .Input(self.Input())
- .Members()
- .Add(fieldNodes)
- .Build()
- .Done()
- .Ptr();
- }
-
- return Build<TCoCombineByKey>(ctx, self.Pos())
- .Input(newInput)
- .PreMapLambda(ctx.DeepCopyLambda(self.PreMapLambda().Ref()))
- .KeySelectorLambda(ctx.DeepCopyLambda(self.KeySelectorLambda().Ref()))
- .InitHandlerLambda(ctx.DeepCopyLambda(self.InitHandlerLambda().Ref()))
- .UpdateHandlerLambda(ctx.DeepCopyLambda(self.UpdateHandlerLambda().Ref()))
- .FinishHandlerLambda(ctx.DeepCopyLambda(self.FinishHandlerLambda().Ref()))
- .Done()
- .Ptr();
- };
-
+ return node;
+ }
+
+ auto itemArg = self.PreMapLambda().Args().Arg(0);
+ auto itemType = itemArg.Ref().GetTypeAnn();
+ if (itemType->GetKind() != ETypeAnnotationKind::Struct) {
+ return node;
+ }
+
+ auto itemStructType = itemType->Cast<TStructExprType>();
+ if (itemStructType->GetSize() == 0) {
+ return node;
+ }
+
+ TSet<TStringBuf> usedFields;
+ if (!HaveFieldsSubset(self.PreMapLambda().Body().Ptr(), itemArg.Ref(), usedFields, *optCtx.ParentsMap)) {
+ return node;
+ }
+
+ TExprNode::TPtr newInput;
+ if (self.Input().Ref().IsCallable("Take") || self.Input().Ref().IsCallable("Skip") || self.Input().Maybe<TCoExtendBase>()) {
+ TSet<TString> fields;
+ for (auto& x : usedFields) {
+ fields.emplace(TString(x));
+ }
+
+ TExprNode::TListType filteredInputs;
+ for (ui32 index = 0; index < self.Input().Ref().ChildrenSize(); ++index) {
+ auto x = self.Input().Ref().ChildPtr(index);
+ if (!self.Input().Maybe<TCoExtendBase>() && index > 0) {
+ filteredInputs.push_back(x);
+ continue;
+ }
+
+ filteredInputs.push_back(FilterByFields(node->Pos(), x, fields, ctx, false));
+ }
+
+ YQL_CLOG(DEBUG, Core) << "FieldsSubset in " << node->Content() << " over " << self.Input().Ref().Content();
+ newInput = ctx.ChangeChildren(self.Input().Ref(), std::move(filteredInputs));
+ }
+ else {
+ TExprNode::TListType fieldNodes;
+ for (auto& item : itemStructType->GetItems()) {
+ if (usedFields.contains(item->GetName())) {
+ fieldNodes.push_back(ctx.NewAtom(self.Pos(), item->GetName()));
+ }
+ }
+
+ YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
+ newInput = Build<TCoExtractMembers>(ctx, self.Input().Pos())
+ .Input(self.Input())
+ .Members()
+ .Add(fieldNodes)
+ .Build()
+ .Done()
+ .Ptr();
+ }
+
+ return Build<TCoCombineByKey>(ctx, self.Pos())
+ .Input(newInput)
+ .PreMapLambda(ctx.DeepCopyLambda(self.PreMapLambda().Ref()))
+ .KeySelectorLambda(ctx.DeepCopyLambda(self.KeySelectorLambda().Ref()))
+ .InitHandlerLambda(ctx.DeepCopyLambda(self.InitHandlerLambda().Ref()))
+ .UpdateHandlerLambda(ctx.DeepCopyLambda(self.UpdateHandlerLambda().Ref()))
+ .FinishHandlerLambda(ctx.DeepCopyLambda(self.FinishHandlerLambda().Ref()))
+ .Done()
+ .Ptr();
+ };
+
map["EquiJoin"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
ui32 inputsCount = node->ChildrenSize() - 2;
for (ui32 i = 0; i < inputsCount; ++i) {
@@ -1826,94 +1826,94 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
}
if (self.Input().Maybe<TCoTake>()) {
- if (auto res = ApplyExtractMembersToTake(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
- return res;
- }
- return node;
+ if (auto res = ApplyExtractMembersToTake(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
+ return res;
+ }
+ return node;
}
if (self.Input().Maybe<TCoSkip>()) {
- if (auto res = ApplyExtractMembersToSkip(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
- return res;
- }
- return node;
+ if (auto res = ApplyExtractMembersToSkip(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
+ return res;
+ }
+ return node;
}
if (self.Input().Maybe<TCoSkipNullMembers>()) {
- if (auto res = ApplyExtractMembersToSkipNullMembers(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
- return res;
- }
- return node;
- }
-
- if (self.Input().Maybe<TCoFilterNullMembers>()) {
- if (auto res = ApplyExtractMembersToFilterNullMembers(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
- return res;
- }
- return node;
- }
-
- if (self.Input().Maybe<TCoSortBase>()) {
- if (auto res = ApplyExtractMembersToSort(self.Input().Ptr(), self.Members().Ptr(), *optCtx.ParentsMap, ctx, {})) {
- return res;
- }
- return node;
- }
-
- if (self.Input().Maybe<TCoAssumeUnique>()) {
- if (auto res = ApplyExtractMembersToAssumeUnique(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
- return res;
- }
- return node;
- }
-
- if (self.Input().Maybe<TCoTopBase>()) {
- if (auto res = ApplyExtractMembersToTop(self.Input().Ptr(), self.Members().Ptr(), *optCtx.ParentsMap, ctx, {})) {
- return res;
- }
- return node;
- }
-
- if (self.Input().Maybe<TCoExtendBase>()) {
- if (auto res = ApplyExtractMembersToExtend(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
- return res;
- }
- return node;
+ if (auto res = ApplyExtractMembersToSkipNullMembers(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
+ return res;
+ }
+ return node;
+ }
+
+ if (self.Input().Maybe<TCoFilterNullMembers>()) {
+ if (auto res = ApplyExtractMembersToFilterNullMembers(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
+ return res;
+ }
+ return node;
+ }
+
+ if (self.Input().Maybe<TCoSortBase>()) {
+ if (auto res = ApplyExtractMembersToSort(self.Input().Ptr(), self.Members().Ptr(), *optCtx.ParentsMap, ctx, {})) {
+ return res;
+ }
+ return node;
+ }
+
+ if (self.Input().Maybe<TCoAssumeUnique>()) {
+ if (auto res = ApplyExtractMembersToAssumeUnique(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
+ return res;
+ }
+ return node;
+ }
+
+ if (self.Input().Maybe<TCoTopBase>()) {
+ if (auto res = ApplyExtractMembersToTop(self.Input().Ptr(), self.Members().Ptr(), *optCtx.ParentsMap, ctx, {})) {
+ return res;
+ }
+ return node;
+ }
+
+ if (self.Input().Maybe<TCoExtendBase>()) {
+ if (auto res = ApplyExtractMembersToExtend(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
+ return res;
+ }
+ return node;
}
if (self.Input().Maybe<TCoEquiJoin>()) {
- if (auto res = ApplyExtractMembersToEquiJoin(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
- return res;
+ if (auto res = ApplyExtractMembersToEquiJoin(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
+ return res;
}
- return node;
+ return node;
}
- if (self.Input().Maybe<TCoFlatMapBase>()) {
- if (auto res = ApplyExtractMembersToFlatMap(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
- return res;
+ if (self.Input().Maybe<TCoFlatMapBase>()) {
+ if (auto res = ApplyExtractMembersToFlatMap(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
+ return res;
}
- return node;
+ return node;
}
if (self.Input().Maybe<TCoPartitionByKey>()) {
- if (auto res = ApplyExtractMembersToPartitionByKey(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
- return res;
- }
- return node;
+ if (auto res = ApplyExtractMembersToPartitionByKey(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
+ return res;
+ }
+ return node;
}
if (self.Input().Maybe<TCoCalcOverWindowBase>() || self.Input().Maybe<TCoCalcOverWindowGroup>()) {
- if (auto res = ApplyExtractMembersToCalcOverWindow(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
- return res;
+ if (auto res = ApplyExtractMembersToCalcOverWindow(self.Input().Ptr(), self.Members().Ptr(), ctx, {})) {
+ return res;
}
- return node;
+ return node;
}
if (self.Input().Maybe<TCoAggregate>()) {
- if (auto res = ApplyExtractMembersToAggregate(self.Input().Ptr(), self.Members().Ptr(), *optCtx.ParentsMap, ctx, {})) {
- return res;
+ if (auto res = ApplyExtractMembersToAggregate(self.Input().Ptr(), self.Members().Ptr(), *optCtx.ParentsMap, ctx, {})) {
+ return res;
}
- return node;
+ return node;
}
if (self.Input().Maybe<TCoChopper>()) {
@@ -1941,16 +1941,16 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
};
map[TCoChopper::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- const TCoChopper chopper(node);
- const auto arg = chopper.Handler().Args().Arg(1).Raw();
- if (const auto parents = optCtx.ParentsMap->find(arg); parents != optCtx.ParentsMap->cend()
- && parents->second.size() == 1
- && (*parents->second.begin())->IsCallable(TCoExtractMembers::CallableName())
- && arg == &(*parents->second.begin())->Head())
- {
- const auto extract = *parents->second.begin();
+ const TCoChopper chopper(node);
+ const auto arg = chopper.Handler().Args().Arg(1).Raw();
+ if (const auto parents = optCtx.ParentsMap->find(arg); parents != optCtx.ParentsMap->cend()
+ && parents->second.size() == 1
+ && (*parents->second.begin())->IsCallable(TCoExtractMembers::CallableName())
+ && arg == &(*parents->second.begin())->Head())
+ {
+ const auto extract = *parents->second.begin();
std::map<std::string_view, TExprNode::TPtr> usedFields;
- auto fields = extract->Tail().ChildrenList();
+ auto fields = extract->Tail().ChildrenList();
std::for_each(fields.cbegin(), fields.cend(), [&](const TExprNode::TPtr& field){ usedFields.emplace(field->Content(), field); });
if (HaveFieldsSubset(chopper.KeyExtractor().Body().Ptr(), chopper.KeyExtractor().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false) && !usedFields.empty() &&
@@ -1963,7 +1963,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
[](std::pair<const std::string_view, TExprNode::TPtr>& item){ return std::move(item.second); });
}
- YQL_CLOG(DEBUG, Core) << "Pull out " << extract->Content() << " from " << node->Content();
+ YQL_CLOG(DEBUG, Core) << "Pull out " << extract->Content() << " from " << node->Content();
return Build<TCoChopper>(ctx, chopper.Pos())
.Input<TCoExtractMembers>()
.Input(chopper.Input())
@@ -1979,7 +1979,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
};
map["WindowTraits"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- auto structType = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
+ auto structType = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TStructExprType>();
TSet<TStringBuf> usedFields;
auto initLambda = node->Child(1);
auto updateLambda = node->Child(2);
@@ -2056,7 +2056,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
};
map["AggregationTraits"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- auto type = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() != ETypeAnnotationKind::Struct) {
// usually distinct, type of column is used instead
return node;
@@ -2107,7 +2107,7 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
map["SessionWindowTraits"] = map["SortTraits"] = map["Lag"] = map["Lead"] = map["RowNumber"] = map["Rank"] = map["DenseRank"] =
[](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx)
{
- auto structType = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()
+ auto structType = node->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType()
->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
if (node->IsCallable("RowNumber")) {
if (structType->GetSize() == 0) {
@@ -2233,75 +2233,75 @@ void RegisterCoFlowCallables2(TCallableOptimizerMap& map) {
return RebuildCalcOverWindowGroup(node->Child(0)->Pos(), node->Child(0)->ChildPtr(0), calcs, ctx);
};
-
- map[TCoCondense::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- TCoCondense self(node);
- if (!optCtx.IsSingleUsage(self.Input().Ref())) {
- return node;
- }
-
- std::map<std::string_view, TExprNode::TPtr> usedFields;
- if (HaveFieldsSubset(self.SwitchHandler().Body().Ptr(), self.SwitchHandler().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
- && !usedFields.empty()
- && HaveFieldsSubset(self.UpdateHandler().Body().Ptr(), self.UpdateHandler().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
- && !usedFields.empty()
- && usedFields.size() < GetSeqItemType(self.Input().Ref().GetTypeAnn())->Cast<TStructExprType>()->GetSize())
- {
- TExprNode::TListType fields;
- fields.reserve(usedFields.size());
- std::transform(usedFields.begin(), usedFields.end(), std::back_inserter(fields),
- [](std::pair<const std::string_view, TExprNode::TPtr>& item){ return std::move(item.second); });
-
- YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
- return Build<TCoCondense>(ctx, node->Pos())
- .Input<TCoExtractMembers>()
- .Input(self.Input())
- .Members()
- .Add(std::move(fields))
- .Build()
- .Build()
- .SwitchHandler(ctx.DeepCopyLambda(self.SwitchHandler().Ref()))
- .UpdateHandler(ctx.DeepCopyLambda(self.UpdateHandler().Ref()))
- .Done().Ptr();
- }
- return node;
- };
-
- map[TCoCondense1::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- TCoCondense1 self(node);
- if (!optCtx.IsSingleUsage(self.Input().Ref())) {
- return node;
- }
-
- std::map<std::string_view, TExprNode::TPtr> usedFields;
- if (HaveFieldsSubset(self.InitHandler().Body().Ptr(), self.InitHandler().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
- && !usedFields.empty()
- && HaveFieldsSubset(self.SwitchHandler().Body().Ptr(), self.SwitchHandler().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
- && !usedFields.empty()
- && HaveFieldsSubset(self.UpdateHandler().Body().Ptr(), self.UpdateHandler().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
- && !usedFields.empty()
- && usedFields.size() < GetSeqItemType(self.Input().Ref().GetTypeAnn())->Cast<TStructExprType>()->GetSize())
- {
- TExprNode::TListType fields;
- fields.reserve(usedFields.size());
- std::transform(usedFields.begin(), usedFields.end(), std::back_inserter(fields),
- [](std::pair<const std::string_view, TExprNode::TPtr>& item){ return std::move(item.second); });
-
- YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
- return Build<TCoCondense1>(ctx, node->Pos())
- .Input<TCoExtractMembers>()
- .Input(self.Input())
- .Members()
- .Add(std::move(fields))
- .Build()
- .Build()
- .InitHandler(ctx.DeepCopyLambda(self.InitHandler().Ref()))
- .SwitchHandler(ctx.DeepCopyLambda(self.SwitchHandler().Ref()))
- .UpdateHandler(ctx.DeepCopyLambda(self.UpdateHandler().Ref()))
- .Done().Ptr();
- }
- return node;
- };
+
+ map[TCoCondense::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ TCoCondense self(node);
+ if (!optCtx.IsSingleUsage(self.Input().Ref())) {
+ return node;
+ }
+
+ std::map<std::string_view, TExprNode::TPtr> usedFields;
+ if (HaveFieldsSubset(self.SwitchHandler().Body().Ptr(), self.SwitchHandler().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
+ && !usedFields.empty()
+ && HaveFieldsSubset(self.UpdateHandler().Body().Ptr(), self.UpdateHandler().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
+ && !usedFields.empty()
+ && usedFields.size() < GetSeqItemType(self.Input().Ref().GetTypeAnn())->Cast<TStructExprType>()->GetSize())
+ {
+ TExprNode::TListType fields;
+ fields.reserve(usedFields.size());
+ std::transform(usedFields.begin(), usedFields.end(), std::back_inserter(fields),
+ [](std::pair<const std::string_view, TExprNode::TPtr>& item){ return std::move(item.second); });
+
+ YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
+ return Build<TCoCondense>(ctx, node->Pos())
+ .Input<TCoExtractMembers>()
+ .Input(self.Input())
+ .Members()
+ .Add(std::move(fields))
+ .Build()
+ .Build()
+ .SwitchHandler(ctx.DeepCopyLambda(self.SwitchHandler().Ref()))
+ .UpdateHandler(ctx.DeepCopyLambda(self.UpdateHandler().Ref()))
+ .Done().Ptr();
+ }
+ return node;
+ };
+
+ map[TCoCondense1::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ TCoCondense1 self(node);
+ if (!optCtx.IsSingleUsage(self.Input().Ref())) {
+ return node;
+ }
+
+ std::map<std::string_view, TExprNode::TPtr> usedFields;
+ if (HaveFieldsSubset(self.InitHandler().Body().Ptr(), self.InitHandler().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
+ && !usedFields.empty()
+ && HaveFieldsSubset(self.SwitchHandler().Body().Ptr(), self.SwitchHandler().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
+ && !usedFields.empty()
+ && HaveFieldsSubset(self.UpdateHandler().Body().Ptr(), self.UpdateHandler().Args().Arg(0).Ref(), usedFields, *optCtx.ParentsMap, false)
+ && !usedFields.empty()
+ && usedFields.size() < GetSeqItemType(self.Input().Ref().GetTypeAnn())->Cast<TStructExprType>()->GetSize())
+ {
+ TExprNode::TListType fields;
+ fields.reserve(usedFields.size());
+ std::transform(usedFields.begin(), usedFields.end(), std::back_inserter(fields),
+ [](std::pair<const std::string_view, TExprNode::TPtr>& item){ return std::move(item.second); });
+
+ YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
+ return Build<TCoCondense1>(ctx, node->Pos())
+ .Input<TCoExtractMembers>()
+ .Input(self.Input())
+ .Members()
+ .Add(std::move(fields))
+ .Build()
+ .Build()
+ .InitHandler(ctx.DeepCopyLambda(self.InitHandler().Ref()))
+ .SwitchHandler(ctx.DeepCopyLambda(self.SwitchHandler().Ref()))
+ .UpdateHandler(ctx.DeepCopyLambda(self.UpdateHandler().Ref()))
+ .Done().Ptr();
+ }
+ return node;
+ };
}
}
diff --git a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp
index a32a6c3dab..cfb56ba329 100644
--- a/ydb/library/yql/core/common_opt/yql_co_simple1.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_simple1.cpp
@@ -11,13 +11,13 @@
#include <ydb/library/yql/utils/log/log.h>
#include <util/generic/map.h>
-#include <util/generic/bitmap.h>
-#include <util/string/cast.h>
+#include <util/generic/bitmap.h>
+#include <util/string/cast.h>
#include <util/generic/xrange.h>
-
-#include <algorithm>
-#include <iterator>
-
+
+#include <algorithm>
+#include <iterator>
+
namespace NYql {
namespace {
@@ -36,84 +36,84 @@ public:
TInt operator() (TInt cur, TInt value) { return std::max(cur, value); }
};
-bool CanRewriteToEmptyContainer(const TExprNode& src) {
- if (auto multi = src.GetConstraint<TMultiConstraintNode>()) {
- for (auto& item: multi->GetItems()) {
- for (auto c: item.second.GetAllConstraints()) {
- if (c->GetName() != TEmptyConstraintNode::Name()) {
- return false;
- }
- }
- }
- }
- return true;
-}
-
-TExprNode::TPtr KeepSortedConstraint(TExprNode::TPtr node, const TSortedConstraintNode* sorted, TExprContext& ctx) {
- if (!sorted) {
- return node;
- }
+bool CanRewriteToEmptyContainer(const TExprNode& src) {
+ if (auto multi = src.GetConstraint<TMultiConstraintNode>()) {
+ for (auto& item: multi->GetItems()) {
+ for (auto c: item.second.GetAllConstraints()) {
+ if (c->GetName() != TEmptyConstraintNode::Name()) {
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+}
+
+TExprNode::TPtr KeepSortedConstraint(TExprNode::TPtr node, const TSortedConstraintNode* sorted, TExprContext& ctx) {
+ if (!sorted) {
+ return node;
+ }
const auto& constent = sorted->GetContent();
- return ctx.Builder(node->Pos())
- .Callable("AssumeSorted")
- .Add(0, std::move(node))
- .List(1)
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- size_t index = 0;
+ return ctx.Builder(node->Pos())
+ .Callable("AssumeSorted")
+ .Add(0, std::move(node))
+ .List(1)
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ size_t index = 0;
for (auto c : constent) {
- parent.Callable(index++, "Bool")
+ parent.Callable(index++, "Bool")
.Atom(0, ToString(c.second), TNodeFlags::Default)
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Lambda(2)
- .Param("item")
- .List()
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- size_t index = 0;
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Lambda(2)
+ .Param("item")
+ .List()
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ size_t index = 0;
for (auto c : constent) {
- parent.Callable(index++, "Member")
- .Arg(0, "item")
+ parent.Callable(index++, "Member")
+ .Arg(0, "item")
.Atom(1, c.first.front())
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Seal()
- .Seal()
- .Build();
-}
-
-TExprNode::TPtr KeepConstraints(TExprNode::TPtr node, const TExprNode& src, TExprContext& ctx) {
- auto res = KeepSortedConstraint(node, src.GetConstraint<TSortedConstraintNode>(), ctx);
- if (auto uniq = src.GetConstraint<TUniqueConstraintNode>()) {
- res = ctx.Builder(node->Pos())
- .Callable("AssumeUnique")
- .Add(0, std::move(res))
- .List(1)
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- size_t index = 0;
- for (auto col: uniq->GetColumns()) {
- parent.Atom(index++, ToString(col), TNodeFlags::Default);
- }
- return parent;
- })
- .Seal()
- .Seal()
- .Build();
- }
- return res;
-}
-
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+}
+
+TExprNode::TPtr KeepConstraints(TExprNode::TPtr node, const TExprNode& src, TExprContext& ctx) {
+ auto res = KeepSortedConstraint(node, src.GetConstraint<TSortedConstraintNode>(), ctx);
+ if (auto uniq = src.GetConstraint<TUniqueConstraintNode>()) {
+ res = ctx.Builder(node->Pos())
+ .Callable("AssumeUnique")
+ .Add(0, std::move(res))
+ .List(1)
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ size_t index = 0;
+ for (auto col: uniq->GetColumns()) {
+ parent.Atom(index++, ToString(col), TNodeFlags::Default);
+ }
+ return parent;
+ })
+ .Seal()
+ .Seal()
+ .Build();
+ }
+ return res;
+}
+
template<typename TInt>
bool ConstIntAggregate(const TExprNode::TChildrenType& values, std::function<TInt(TInt, TInt)> aggFunc,
TInt& result)
{
auto extractValue = [&values] (size_t index, TInt& value) {
- if (!TMaybeNode<TCoIntegralCtor>(values[index])) {
+ if (!TMaybeNode<TCoIntegralCtor>(values[index])) {
return false;
}
@@ -146,17 +146,17 @@ bool ConstIntAggregate(const TExprNode::TChildrenType& values, std::function<TIn
template<template<typename> class TAgg>
TExprNode::TPtr ConstFoldNodeIntAggregate(const TExprNode::TPtr& node, TExprContext& ctx) {
- auto dataSlot = node->GetTypeAnn()->Cast<TDataExprType>()->GetSlot();
+ auto dataSlot = node->GetTypeAnn()->Cast<TDataExprType>()->GetSlot();
if (dataSlot == EDataSlot::Uint64) {
ui64 result;
if (ConstIntAggregate<ui64>(node->Children(), TAgg<ui64>(), result)) {
- return ctx.NewCallable(node->Pos(), node->GetTypeAnn()->Cast<TDataExprType>()->GetName(), {ctx.NewAtom(node->Pos(), ToString(result))});
+ return ctx.NewCallable(node->Pos(), node->GetTypeAnn()->Cast<TDataExprType>()->GetName(), {ctx.NewAtom(node->Pos(), ToString(result))});
}
} else {
i64 result;
if (ConstIntAggregate<i64>(node->Children(), TAgg<i64>(), result)) {
- return ctx.NewCallable(node->Pos(), node->GetTypeAnn()->Cast<TDataExprType>()->GetName(), {ctx.NewAtom(node->Pos(), ToString(result))});
+ return ctx.NewCallable(node->Pos(), node->GetTypeAnn()->Cast<TDataExprType>()->GetName(), {ctx.NewAtom(node->Pos(), ToString(result))});
}
}
@@ -227,7 +227,7 @@ TExprNode::TPtr ExpandFlattenEquiJoin(const TExprNode::TPtr& node, TExprContext&
for (ui32 idx = 0; idx < numLists; ++idx) {
const auto& listPair = *node->Child(idx);
const auto& list = listPair.Head();
- const TTypeAnnotationNode* itemType = list.GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ const TTypeAnnotationNode* itemType = list.GetTypeAnn()->Cast<TListExprType>()->GetItemType();
auto structType = itemType->Cast<TStructExprType>();
if (auto err = labels.Add(ctx, *listPair.Child(1), structType)) {
ctx.AddError(*err);
@@ -451,29 +451,29 @@ TExprNode::TPtr RemoveDeadPayloadColumns(const TExprNode::TPtr& node, TExprConte
return ctx.ChangeChildren(*node, std::move(nodeChildren));
}
-TExprNode::TPtr HandleEmptyListInJoin(const TExprNode::TPtr& node, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) {
- TMaybe<TJoinLabels> labels;
+TExprNode::TPtr HandleEmptyListInJoin(const TExprNode::TPtr& node, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) {
+ TMaybe<TJoinLabels> labels;
for (ui32 inputIndex = 0; inputIndex < node->ChildrenSize() - 2; ++inputIndex) {
auto& input = SkipCallables(node->Child(inputIndex)->Head(), SkippableCallables);
- if (!IsEmptyContainer(input) && !IsEmpty(input, typeCtx)) {
+ if (!IsEmptyContainer(input) && !IsEmpty(input, typeCtx)) {
continue;
}
auto joinTree = node->Child(node->ChildrenSize() - 2);
- if (!labels) {
- labels.ConstructInPlace();
- for (ui32 i = 0; i < node->ChildrenSize() - 2; ++i) {
- auto err = labels->Add(ctx, *node->Child(i)->Child(1),
- node->Child(i)->Head().GetTypeAnn()->Cast<TListExprType>()
- ->GetItemType()->Cast<TStructExprType>());
- if (err) {
- ctx.AddError(*err);
- return nullptr;
- }
+ if (!labels) {
+ labels.ConstructInPlace();
+ for (ui32 i = 0; i < node->ChildrenSize() - 2; ++i) {
+ auto err = labels->Add(ctx, *node->Child(i)->Child(1),
+ node->Child(i)->Head().GetTypeAnn()->Cast<TListExprType>()
+ ->GetItemType()->Cast<TStructExprType>());
+ if (err) {
+ ctx.AddError(*err);
+ return nullptr;
+ }
}
}
- if (IsRequiredSide(joinTree, *labels, inputIndex).first) {
+ if (IsRequiredSide(joinTree, *labels, inputIndex).first) {
return ctx.NewCallable(node->Pos(), "List", {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
}
}
@@ -481,99 +481,99 @@ TExprNode::TPtr HandleEmptyListInJoin(const TExprNode::TPtr& node, TExprContext&
return node;
}
-TExprNode::TPtr UpdateJoinTreeUniqueRecursive(const TExprNode::TPtr& joinTree, const TJoinLabels& labels, const TVector<const TUniqueConstraintNode*>& unique, TExprContext& ctx) {
- TExprNode::TPtr res = joinTree;
-
- TEquiJoinLinkSettings linkSettings = GetEquiJoinLinkSettings(*joinTree->Child(5));
- bool updateSettings = false;
- auto left = joinTree->ChildPtr(1);
- if (!left->IsAtom()) {
- left = UpdateJoinTreeUniqueRecursive(left, labels, unique, ctx);
- if (left != joinTree->ChildPtr(1)) {
- res = ctx.ChangeChild(*res, 1, std::move(left));
- }
- } else if (linkSettings.LeftHints.find("unique") == linkSettings.LeftHints.end()) {
- if (auto label = labels.FindInput(left->Content())) {
- if (auto ndx = labels.FindInputIndex(left->Content())) {
- if (auto u = unique[*ndx]) {
- auto keys = joinTree->Child(3);
- THashSet<TString> keySet;
- for (ui32 i = 0; i < keys->ChildrenSize(); i += 2) {
- keySet.insert((*label)->MemberName(keys->Child(i)->Content(), keys->Child(i + 1)->Content()));
- }
- if (AllOf(u->GetColumns(), [&keySet](TStringBuf col) { return keySet.find(col) != keySet.end(); })) {
- linkSettings.LeftHints.insert("unique");
- updateSettings = true;
- }
- }
- }
- }
- }
-
- auto right = joinTree->ChildPtr(2);
- if (!right->IsAtom()) {
- right = UpdateJoinTreeUniqueRecursive(right, labels, unique, ctx);
- if (right != joinTree->ChildPtr(2)) {
- res = ctx.ChangeChild(*res, 2, std::move(right));
- }
- } else if (linkSettings.RightHints.find("unique") == linkSettings.RightHints.end()) {
- if (auto label = labels.FindInput(right->Content())) {
- if (auto ndx = labels.FindInputIndex(right->Content())) {
- if (auto u = unique[*ndx]) {
- auto keys = joinTree->Child(4);
- THashSet<TString> keySet;
- for (ui32 i = 0; i < keys->ChildrenSize(); i += 2) {
- keySet.insert((*label)->MemberName(keys->Child(i)->Content(), keys->Child(i + 1)->Content()));
- }
- if (AllOf(u->GetColumns(), [&keySet](TStringBuf col) { return keySet.find(col) != keySet.end(); })) {
- linkSettings.RightHints.insert("unique");
- updateSettings = true;
- }
- }
- }
- }
- }
-
- if (updateSettings) {
- res = ctx.ChangeChild(*res, 5, BuildEquiJoinLinkSettings(linkSettings, ctx));
- }
-
- return res;
-}
-
-
-TExprNode::TPtr HandleUniqueListInJoin(const TExprNode::TPtr& node, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) {
- if (!typeCtx.IsConstraintCheckEnabled<TUniqueConstraintNode>()) {
- return node;
- }
-
- TJoinLabels labels;
- TVector<const TUniqueConstraintNode*> unique;
- unique.reserve(node->ChildrenSize() - 2);
- for (ui32 i = 0; i < node->ChildrenSize() - 2; ++i) {
- auto err = labels.Add(ctx, *node->Child(i)->Child(1),
- node->Child(i)->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>());
- if (err) {
- ctx.AddError(*err);
- return nullptr;
- }
- unique.push_back(node->Child(i)->Head().GetConstraint<TUniqueConstraintNode>());
- }
-
- auto joinTree = UpdateJoinTreeUniqueRecursive(node->ChildPtr(node->ChildrenSize() - 2), labels, unique, ctx);
- if (joinTree != node->ChildPtr(node->ChildrenSize() - 2)) {
- return ctx.ChangeChild(*node, node->ChildrenSize() - 2, std::move(joinTree));
- }
-
- return node;
-}
-
+TExprNode::TPtr UpdateJoinTreeUniqueRecursive(const TExprNode::TPtr& joinTree, const TJoinLabels& labels, const TVector<const TUniqueConstraintNode*>& unique, TExprContext& ctx) {
+ TExprNode::TPtr res = joinTree;
+
+ TEquiJoinLinkSettings linkSettings = GetEquiJoinLinkSettings(*joinTree->Child(5));
+ bool updateSettings = false;
+ auto left = joinTree->ChildPtr(1);
+ if (!left->IsAtom()) {
+ left = UpdateJoinTreeUniqueRecursive(left, labels, unique, ctx);
+ if (left != joinTree->ChildPtr(1)) {
+ res = ctx.ChangeChild(*res, 1, std::move(left));
+ }
+ } else if (linkSettings.LeftHints.find("unique") == linkSettings.LeftHints.end()) {
+ if (auto label = labels.FindInput(left->Content())) {
+ if (auto ndx = labels.FindInputIndex(left->Content())) {
+ if (auto u = unique[*ndx]) {
+ auto keys = joinTree->Child(3);
+ THashSet<TString> keySet;
+ for (ui32 i = 0; i < keys->ChildrenSize(); i += 2) {
+ keySet.insert((*label)->MemberName(keys->Child(i)->Content(), keys->Child(i + 1)->Content()));
+ }
+ if (AllOf(u->GetColumns(), [&keySet](TStringBuf col) { return keySet.find(col) != keySet.end(); })) {
+ linkSettings.LeftHints.insert("unique");
+ updateSettings = true;
+ }
+ }
+ }
+ }
+ }
+
+ auto right = joinTree->ChildPtr(2);
+ if (!right->IsAtom()) {
+ right = UpdateJoinTreeUniqueRecursive(right, labels, unique, ctx);
+ if (right != joinTree->ChildPtr(2)) {
+ res = ctx.ChangeChild(*res, 2, std::move(right));
+ }
+ } else if (linkSettings.RightHints.find("unique") == linkSettings.RightHints.end()) {
+ if (auto label = labels.FindInput(right->Content())) {
+ if (auto ndx = labels.FindInputIndex(right->Content())) {
+ if (auto u = unique[*ndx]) {
+ auto keys = joinTree->Child(4);
+ THashSet<TString> keySet;
+ for (ui32 i = 0; i < keys->ChildrenSize(); i += 2) {
+ keySet.insert((*label)->MemberName(keys->Child(i)->Content(), keys->Child(i + 1)->Content()));
+ }
+ if (AllOf(u->GetColumns(), [&keySet](TStringBuf col) { return keySet.find(col) != keySet.end(); })) {
+ linkSettings.RightHints.insert("unique");
+ updateSettings = true;
+ }
+ }
+ }
+ }
+ }
+
+ if (updateSettings) {
+ res = ctx.ChangeChild(*res, 5, BuildEquiJoinLinkSettings(linkSettings, ctx));
+ }
+
+ return res;
+}
+
+
+TExprNode::TPtr HandleUniqueListInJoin(const TExprNode::TPtr& node, TExprContext& ctx, const TTypeAnnotationContext& typeCtx) {
+ if (!typeCtx.IsConstraintCheckEnabled<TUniqueConstraintNode>()) {
+ return node;
+ }
+
+ TJoinLabels labels;
+ TVector<const TUniqueConstraintNode*> unique;
+ unique.reserve(node->ChildrenSize() - 2);
+ for (ui32 i = 0; i < node->ChildrenSize() - 2; ++i) {
+ auto err = labels.Add(ctx, *node->Child(i)->Child(1),
+ node->Child(i)->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>());
+ if (err) {
+ ctx.AddError(*err);
+ return nullptr;
+ }
+ unique.push_back(node->Child(i)->Head().GetConstraint<TUniqueConstraintNode>());
+ }
+
+ auto joinTree = UpdateJoinTreeUniqueRecursive(node->ChildPtr(node->ChildrenSize() - 2), labels, unique, ctx);
+ if (joinTree != node->ChildPtr(node->ChildrenSize() - 2)) {
+ return ctx.ChangeChild(*node, node->ChildrenSize() - 2, std::move(joinTree));
+ }
+
+ return node;
+}
+
bool IsDataType(const TTypeAnnotationNode& type) {
return type.GetKind() == ETypeAnnotationKind::Data;
}
bool IsDataType(const TExprNode& node) {
- return node.GetTypeAnn() && IsDataType(*node.GetTypeAnn());
+ return node.GetTypeAnn() && IsDataType(*node.GetTypeAnn());
}
bool IsBoolType(const TTypeAnnotationNode& type) {
@@ -582,7 +582,7 @@ bool IsBoolType(const TTypeAnnotationNode& type) {
}
bool IsBoolType(const TExprNode& node) {
- return node.GetTypeAnn() && IsBoolType(*node.GetTypeAnn());
+ return node.GetTypeAnn() && IsBoolType(*node.GetTypeAnn());
}
bool IsOptBoolType(const TTypeAnnotationNode& type) {
@@ -591,13 +591,13 @@ bool IsOptBoolType(const TTypeAnnotationNode& type) {
}
bool IsOptBoolType(const TExprNode& node) {
- return node.GetTypeAnn() && IsOptBoolType(*node.GetTypeAnn());
+ return node.GetTypeAnn() && IsOptBoolType(*node.GetTypeAnn());
}
template <bool AppendOrPrepend>
-TExprNode::TPtr OptimizeInsert(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+TExprNode::TPtr OptimizeInsert(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
const auto& list = AppendOrPrepend ? node->Head() : node->Tail();
- if (IsEmptyContainer(list) || IsEmpty(list, *optCtx.Types)) {
+ if (IsEmptyContainer(list) || IsEmpty(list, *optCtx.Types)) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over empty " << list.Content();
return ctx.NewCallable(node->Pos(), "AsList", {AppendOrPrepend ? node->TailPtr() : node->HeadPtr()});
}
@@ -654,7 +654,7 @@ TExprNode::TPtr ExpandUnionAll(const TExprNode::TPtr& node, TExprContext& ctx, T
return node->HeadPtr();
}
- auto resultStructType = node->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
+ auto resultStructType = node->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
TVector<TExprNode::TPtr> nulls(resultStructType->GetSize());
auto remapList = [&ctx, &nulls, resultStructType](TExprNode::TPtr input, const TTypeAnnotationNode* inputType) -> TExprNode::TPtr {
auto pos = input->Pos();
@@ -729,7 +729,7 @@ TExprNode::TPtr ExpandUnionAll(const TExprNode::TPtr& node, TExprContext& ctx, T
if (!remapped) {
return node;
}
-
+
remappedList.push_back(remapped);
}
@@ -1067,12 +1067,12 @@ TExprNode::TPtr OptimizeEquality(const TExprNode::TPtr& node, TExprContext& ctx)
template <bool IsList, bool IsLookup = false>
TExprNode::TPtr OptimizeContains(const TExprNode::TPtr& node, TExprContext& ctx) {
static_assert(!IsList || !IsLookup, "List or Lookup");
- if constexpr (!(IsLookup || IsList)) {
- if (IsDataOrOptionalOfData(node->Head().GetTypeAnn())) {
- return OptimizeEquality<true>(node, ctx);
- }
- }
-
+ if constexpr (!(IsLookup || IsList)) {
+ if (IsDataOrOptionalOfData(node->Head().GetTypeAnn())) {
+ return OptimizeEquality<true>(node, ctx);
+ }
+ }
+
if (const auto& nodeToCheck = SkipCallables(node->Head(), SkippableCallables); nodeToCheck.IsCallable(IsList ? "AsList" : "AsDict")) {
for (ui32 i = 0U; i < nodeToCheck.ChildrenSize(); ++i) {
if ((IsList ? nodeToCheck.Child(i) : &nodeToCheck.Child(i)->Head()) == &node->Tail()) {
@@ -1115,22 +1115,22 @@ TExprNode::TPtr OptimizeContainerIf(const TExprNode::TPtr& node, TExprContext& c
if (node->Head().IsCallable("Bool")) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content() << " '" << node->Head().Head().Content();
const auto value = FromString<bool>(node->Head().Head().Content());
- auto res = value
- ? ctx.NewCallable(node->Tail().Pos(), IsList ? "AsList" : "Just", {node->TailPtr()})
- : //TODO: ctx.NewCallable(node->Head().Pos(), IsList ? "List" : "Nothing", {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)})
- ctx.Builder(node->Head().Pos())
- .Callable(IsList ? "List" : "Nothing")
- .Callable(0, IsList ? "ListType" : "OptionalType")
- .Callable(0, "TypeOf")
- .Add(0, node->TailPtr())
- .Seal()
- .Seal()
- .Seal().Build();
- if (IsList) {
- res = KeepConstraints(res, *node, ctx);
- }
- return res;
-
+ auto res = value
+ ? ctx.NewCallable(node->Tail().Pos(), IsList ? "AsList" : "Just", {node->TailPtr()})
+ : //TODO: ctx.NewCallable(node->Head().Pos(), IsList ? "List" : "Nothing", {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)})
+ ctx.Builder(node->Head().Pos())
+ .Callable(IsList ? "List" : "Nothing")
+ .Callable(0, IsList ? "ListType" : "OptionalType")
+ .Callable(0, "TypeOf")
+ .Add(0, node->TailPtr())
+ .Seal()
+ .Seal()
+ .Seal().Build();
+ if (IsList) {
+ res = KeepConstraints(res, *node, ctx);
+ }
+ return res;
+
}
return node;
}
@@ -1150,29 +1150,29 @@ TExprNode::TPtr OptimizeFlatContainerIf(const TExprNode::TPtr& node, TExprContex
}
const auto& nodeToCheck = SkipCallables(node->Tail(), SkippableCallables);
- if (1U == nodeToCheck.ChildrenSize() && nodeToCheck.IsCallable(IsList ? "AsList" : "Just")) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " with " << nodeToCheck.Content();
- auto res = ctx.NewCallable(node->Pos(), IsList ? "ListIf" : "OptionalIf", {node->HeadPtr(), nodeToCheck.HeadPtr()});
- if (IsList) {
- res = KeepSortedConstraint(res, node->GetConstraint<TSortedConstraintNode>(), ctx);
- }
- return res;
+ if (1U == nodeToCheck.ChildrenSize() && nodeToCheck.IsCallable(IsList ? "AsList" : "Just")) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " with " << nodeToCheck.Content();
+ auto res = ctx.NewCallable(node->Pos(), IsList ? "ListIf" : "OptionalIf", {node->HeadPtr(), nodeToCheck.HeadPtr()});
+ if (IsList) {
+ res = KeepSortedConstraint(res, node->GetConstraint<TSortedConstraintNode>(), ctx);
+ }
+ return res;
}
- if (1U == nodeToCheck.ChildrenSize() && nodeToCheck.IsCallable(IsList ? "List" : "Nothing")) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " with " << nodeToCheck.Content();
- auto res = node->TailPtr();
+ if (1U == nodeToCheck.ChildrenSize() && nodeToCheck.IsCallable(IsList ? "List" : "Nothing")) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " with " << nodeToCheck.Content();
+ auto res = node->TailPtr();
}
if (node->Head().IsCallable("Bool")) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content() << " '" << node->Head().Head().Content();
const auto value = FromString<bool>(node->Head().Head().Content());
- return value
- ? node->TailPtr()
- : KeepConstraints(
- ctx.NewCallable(node->Head().Pos(), IsList ? "List" : "Nothing", {ExpandType(node->Tail().Pos(), *node->GetTypeAnn(), ctx)}),
- *node,
- ctx);
+ return value
+ ? node->TailPtr()
+ : KeepConstraints(
+ ctx.NewCallable(node->Head().Pos(), IsList ? "List" : "Nothing", {ExpandType(node->Tail().Pos(), *node->GetTypeAnn(), ctx)}),
+ *node,
+ ctx);
}
return node;
@@ -1186,13 +1186,13 @@ TExprNode::TPtr OptimizeToOptional(const TExprNode::TPtr& node, TExprContext& ct
}
const auto& nodeToCheck = SkipCallables(node->Head(), SkippableCallables);
- if (nodeToCheck.IsCallable("AsList")) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content();
- return ctx.NewCallable(node->Head().Pos(), "Just", {HeadOrTail ? nodeToCheck.HeadPtr() : nodeToCheck.TailPtr()});
+ if (nodeToCheck.IsCallable("AsList")) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content();
+ return ctx.NewCallable(node->Head().Pos(), "Just", {HeadOrTail ? nodeToCheck.HeadPtr() : nodeToCheck.TailPtr()});
}
if (1U == nodeToCheck.ChildrenSize() && nodeToCheck.IsCallable("List")) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over empty " << nodeToCheck.Content();
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over empty " << nodeToCheck.Content();
return ctx.NewCallable(node->Head().Pos(), "Nothing", {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
}
@@ -1220,8 +1220,8 @@ TExprNode::TPtr OptimizeDirection(const TExprNode::TPtr& node) {
return node;
}
-TExprNode::TPtr OptimizeAsStruct(const TExprNode::TPtr& node, TExprContext& ctx) {
- TExprNode::TPtr singleFrom;
+TExprNode::TPtr OptimizeAsStruct(const TExprNode::TPtr& node, TExprContext& ctx) {
+ TExprNode::TPtr singleFrom;
for (const auto& member : node->Children()) {
if (!member->Child(1)->IsCallable("Member")) {
return node;
@@ -1232,62 +1232,62 @@ TExprNode::TPtr OptimizeAsStruct(const TExprNode::TPtr& node, TExprContext& ctx)
}
auto from = member->Child(1)->HeadPtr();
- if (!singleFrom) {
- if (from->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
+ if (!singleFrom) {
+ if (from->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
return node;
}
- singleFrom = from;
+ singleFrom = from;
} else {
- if (singleFrom != from) {
+ if (singleFrom != from) {
return node;
}
}
}
- if (!singleFrom) {
- return node;
- }
-
- if (singleFrom->GetTypeAnn()->Cast<TStructExprType>()->GetSize() == node->ChildrenSize()) {
- YQL_CLOG(DEBUG, Core) << "CheckClonedStructure";
- return singleFrom;
- }
-
- if (TCoVisit::Match(singleFrom.Get())) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over " << singleFrom->Content();
- return ctx.Builder(node->Pos())
- .Callable("Visit")
- .Add(0, singleFrom->HeadPtr())
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- for (size_t i = 1; i < singleFrom->ChildrenSize(); ++i) {
- auto child = singleFrom->ChildPtr(i);
- if (child->IsAtom()) {
- auto lambda = singleFrom->Child(i + 1);
- parent
- .Add(i, std::move(child))
- .Lambda(i + 1)
- .Param("visitItem")
- .ApplyPartial(lambda->HeadPtr(), node)
- .WithNode(*singleFrom, lambda->TailPtr())
- .With(0, "visitItem")
- .Seal()
- .Seal();
- ++i;
- }
- else {
- parent.ApplyPartial(i, {}, node)
- .WithNode(*singleFrom, std::move(child))
- .Seal();
- }
- }
- return parent;
- })
- .Seal()
- .Build();
- }
-
- return node;
+ if (!singleFrom) {
+ return node;
+ }
+
+ if (singleFrom->GetTypeAnn()->Cast<TStructExprType>()->GetSize() == node->ChildrenSize()) {
+ YQL_CLOG(DEBUG, Core) << "CheckClonedStructure";
+ return singleFrom;
+ }
+
+ if (TCoVisit::Match(singleFrom.Get())) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over " << singleFrom->Content();
+ return ctx.Builder(node->Pos())
+ .Callable("Visit")
+ .Add(0, singleFrom->HeadPtr())
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ for (size_t i = 1; i < singleFrom->ChildrenSize(); ++i) {
+ auto child = singleFrom->ChildPtr(i);
+ if (child->IsAtom()) {
+ auto lambda = singleFrom->Child(i + 1);
+ parent
+ .Add(i, std::move(child))
+ .Lambda(i + 1)
+ .Param("visitItem")
+ .ApplyPartial(lambda->HeadPtr(), node)
+ .WithNode(*singleFrom, lambda->TailPtr())
+ .With(0, "visitItem")
+ .Seal()
+ .Seal();
+ ++i;
+ }
+ else {
+ parent.ApplyPartial(i, {}, node)
+ .WithNode(*singleFrom, std::move(child))
+ .Seal();
+ }
+ }
+ return parent;
+ })
+ .Seal()
+ .Build();
+ }
+
+ return node;
}
TExprNode::TPtr RemoveToStringFromString(const TExprNode::TPtr& node) {
@@ -1309,7 +1309,7 @@ TExprNode::TPtr ConvertMapToFlatmap(TMapType map, TExprContext& ctx) {
.Input(list)
.Lambda()
.Args({ "item" })
- .template Body<TCoJust>()
+ .template Body<TCoJust>()
.template Input<TExprApplier>()
.Apply(lambda)
.With(0, "item")
@@ -1341,7 +1341,7 @@ TExprNode::TPtr ConvertFilterToFlatmap(TFilterType filter, TExprContext& ctx, TO
.Input(list)
.Lambda()
.Args({ "item" })
- .template Body<TCoOptionalIf>()
+ .template Body<TCoOptionalIf>()
.template Predicate<TExprApplier>()
.Apply(lambda)
.With(0, "item")
@@ -1362,12 +1362,12 @@ TExprNode::TPtr ExtractPredicateFromFlatmapOverListIf(const TExprNode& node, TEx
auto ret = ctx.NewCallable(node.Head().Pos(), isOptional ? "FlatOptionalIf" : "FlatListIf",
{ node.Head().HeadPtr(), std::move(item) });
-
- if (isOptional && node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) {
+
+ if (isOptional && node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) {
ret = ctx.NewCallable(node.Head().Pos(), "ToList", { std::move(ret) });
} else if (node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Flow) {
ret = ctx.NewCallable(node.Head().Pos(), "ToFlow", { std::move(ret) });
- } else if (node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream) {
+ } else if (node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Stream) {
ret = ctx.NewCallable(node.Head().Pos(), "ToStream", { std::move(ret) });
}
@@ -2033,8 +2033,8 @@ TExprNode::TPtr BuildEquiJoinForSqlInChain(const TExprNode::TPtr& flatMapNode, c
}
}
- YQL_ENSURE(input->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List);
- auto inputRowType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ YQL_ENSURE(input->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List);
+ auto inputRowType = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
YQL_ENSURE(inputRowType->GetKind() == ETypeAnnotationKind::Struct);
for (const auto& i : inputRowType->Cast<TStructExprType>()->GetItems()) {
@@ -2047,11 +2047,11 @@ TExprNode::TPtr BuildEquiJoinForSqlInChain(const TExprNode::TPtr& flatMapNode, c
.Build();
renames.push_back(rename);
}
- renames.push_back(ctx.Builder(input->Pos())
- .List()
- .Atom(0, "keep_sys")
- .Seal()
- .Build());
+ renames.push_back(ctx.Builder(input->Pos())
+ .List()
+ .Atom(0, "keep_sys")
+ .Seal()
+ .Build());
equiJoinArgs.push_back(joinChain);
equiJoinArgs.push_back(ctx.NewList(input->Pos(), std::move(renames)));
@@ -2097,7 +2097,7 @@ TStringBuf GetEmptyCollectionName(const TTypeAnnotationNode* type) {
}
template <bool Ordered>
-TExprNode::TPtr SimpleFlatMap(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+TExprNode::TPtr SimpleFlatMap(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
const TCoFlatMapBase self(node);
const auto& lambdaBody = self.Lambda().Body().Ref();
const auto& lambdaArg = self.Lambda().Args().Arg(0).Ref();
@@ -2206,20 +2206,20 @@ TExprNode::TPtr SimpleFlatMap(const TExprNode::TPtr& node, TExprContext& ctx, TO
}
}
- if (CanRewriteToEmptyContainer(*node)) {
+ if (CanRewriteToEmptyContainer(*node)) {
const auto& inputToCheck = SkipCallables(node->Head(), SkippableCallables);
- if (IsEmptyContainer(inputToCheck) || IsEmpty(inputToCheck, *optCtx.Types)) {
- YQL_CLOG(DEBUG, Core) << "Empty " << node->Content() << " over " << inputToCheck.Content();
- auto res = ctx.NewCallable(inputToCheck.Pos(), GetEmptyCollectionName(node->GetTypeAnn()), {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
- return KeepConstraints(res, *node, ctx);
- }
+ if (IsEmptyContainer(inputToCheck) || IsEmpty(inputToCheck, *optCtx.Types)) {
+ YQL_CLOG(DEBUG, Core) << "Empty " << node->Content() << " over " << inputToCheck.Content();
+ auto res = ctx.NewCallable(inputToCheck.Pos(), GetEmptyCollectionName(node->GetTypeAnn()), {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
+ return KeepConstraints(res, *node, ctx);
+ }
const auto& lambdaRootToCheck = SkipCallables(node->Tail().Tail(), SkippableCallables);
- if (IsEmptyContainer(lambdaRootToCheck) || IsEmpty(lambdaRootToCheck, *optCtx.Types)) {
- YQL_CLOG(DEBUG, Core) << "Empty " << node->Content() << " with " << lambdaRootToCheck.Content();
- auto res = ctx.NewCallable(lambdaRootToCheck.Pos(), GetEmptyCollectionName(node->GetTypeAnn()), {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
- return KeepConstraints(res, *node, ctx);
- }
+ if (IsEmptyContainer(lambdaRootToCheck) || IsEmpty(lambdaRootToCheck, *optCtx.Types)) {
+ YQL_CLOG(DEBUG, Core) << "Empty " << node->Content() << " with " << lambdaRootToCheck.Content();
+ auto res = ctx.NewCallable(lambdaRootToCheck.Pos(), GetEmptyCollectionName(node->GetTypeAnn()), {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
+ return KeepConstraints(res, *node, ctx);
+ }
}
// rewrite in 'canonical' way (prefer OptionalIf to ListIf)
@@ -2407,8 +2407,8 @@ TExprNode::TPtr OptimizeCollect(const TExprNode::TPtr& node, TExprContext& ctx)
}
const auto& nodeToCheck = SkipCallables(node->Head(), SkippableCallables);
- if (nodeToCheck.IsCallable({node->Content(), "List", "ListIf", "AsList"})) {
- YQL_CLOG(DEBUG, Core) << "Drop " << node->Content() << " over " << nodeToCheck.Content();
+ if (nodeToCheck.IsCallable({node->Content(), "List", "ListIf", "AsList"})) {
+ YQL_CLOG(DEBUG, Core) << "Drop " << node->Content() << " over " << nodeToCheck.Content();
return node->HeadPtr();
}
@@ -2457,8 +2457,8 @@ template <bool TakeOrSkip, bool Inclusive = false>
TExprNode::TPtr OptimizeWhile(const TExprNode::TPtr& node, TExprContext& ctx) {
const auto& emptyCollectionName = GetEmptyCollectionName(node->GetTypeAnn());
const auto& nodeToCheck = SkipCallables(node->Head(), SkippableCallables);
- if (1U == nodeToCheck.ChildrenSize() && nodeToCheck.IsCallable(emptyCollectionName)) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over empty " << nodeToCheck.Content();
+ if (1U == nodeToCheck.ChildrenSize() && nodeToCheck.IsCallable(emptyCollectionName)) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over empty " << nodeToCheck.Content();
return node->HeadPtr();
}
@@ -2466,19 +2466,19 @@ TExprNode::TPtr OptimizeWhile(const TExprNode::TPtr& node, TExprContext& ctx) {
if (lambdaBody.IsCallable("Bool")) {
YQL_CLOG(DEBUG, Core) << node->Content() << " with lambda " << lambdaBody.Content() << " '" << lambdaBody.Head().Content();
const bool isAll = FromString<bool>(lambdaBody.Head().Content());
- return TakeOrSkip == isAll
- ? node->HeadPtr()
- : Inclusive
- ? ctx.Builder(lambdaBody.Pos())
- .Callable("Take")
- .Add(0, node->HeadPtr())
- .Callable(1, "Uint64")
- .Atom(0, "1", TNodeFlags::Default)
- .Seal()
- .Seal().Build()
- : KeepConstraints(
- ctx.NewCallable(lambdaBody.Pos(), emptyCollectionName, {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)}),
- *node, ctx);
+ return TakeOrSkip == isAll
+ ? node->HeadPtr()
+ : Inclusive
+ ? ctx.Builder(lambdaBody.Pos())
+ .Callable("Take")
+ .Add(0, node->HeadPtr())
+ .Callable(1, "Uint64")
+ .Atom(0, "1", TNodeFlags::Default)
+ .Seal()
+ .Seal().Build()
+ : KeepConstraints(
+ ctx.NewCallable(lambdaBody.Pos(), emptyCollectionName, {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)}),
+ *node, ctx);
}
return node;
}
@@ -2524,7 +2524,7 @@ TExprNode::TPtr DropReorder(const TExprNode::TPtr& node, TExprContext& ctx) {
}
template <bool IsTop, bool IsSort>
-TExprNode::TPtr OptimizeReorder(const TExprNode::TPtr& node, TExprContext& ctx) {
+TExprNode::TPtr OptimizeReorder(const TExprNode::TPtr& node, TExprContext& ctx) {
const ui32 ascIndex = node->ChildrenSize() - 2U;
if ((IsSort || IsTop) && 1U == node->Head().ChildrenSize() && node->Head().IsCallable(GetEmptyCollectionName(node->Head().GetTypeAnn()))) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
@@ -2640,7 +2640,7 @@ TExprNode::TPtr OptimizeReorder(const TExprNode::TPtr& node, TExprContext& ctx)
.Seal()
.Build();
- if (topConstr->Equals(*inputConstr)) {
+ if (topConstr->Equals(*inputConstr)) {
return res;
}
@@ -2668,7 +2668,7 @@ TExprNode::TPtr OptimizeReorder(const TExprNode::TPtr& node, TExprContext& ctx)
if (const auto sortConstr = node->GetConstraint<TSortedConstraintNode>()) {
if (sortConstr->IsPrefixOf(*inputConstr)) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over sorted input";
- return KeepSortedConstraint(node->HeadPtr(), sortConstr, ctx);
+ return KeepSortedConstraint(node->HeadPtr(), sortConstr, ctx);
}
}
}
@@ -3300,9 +3300,9 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
auto result = ctx.Builder(node->Pos());
auto cListBuilder = result.Callable("List");
- auto& listBuilder = cListBuilder.Add(0U, ExpandType(node->Pos(), *node->GetTypeAnn(), ctx));
+ auto& listBuilder = cListBuilder.Add(0U, ExpandType(node->Pos(), *node->GetTypeAnn(), ctx));
- const auto structType = ExpandType(node->Pos(), *node->GetTypeAnn()->Cast<TListExprType>()->GetItemType(), ctx);
+ const auto structType = ExpandType(node->Pos(), *node->GetTypeAnn()->Cast<TListExprType>()->GetItemType(), ctx);
ui32 i = 0U;
for (const auto& data : optCtx.Types->Credentials)
for (const auto& cred : *data) {
@@ -3338,10 +3338,10 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
auto result = ctx.Builder(node->Pos());
auto cListBuilder = result.Callable("List");
- auto& listBuilder = cListBuilder.Add(0U, ExpandType(node->Pos(), *node->GetTypeAnn(), ctx));
+ auto& listBuilder = cListBuilder.Add(0U, ExpandType(node->Pos(), *node->GetTypeAnn(), ctx));
- const auto structType = ExpandType(node->Pos(), *node->GetTypeAnn()->Cast<TListExprType>()->GetItemType(), ctx);
- const auto structure = node->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
+ const auto structType = ExpandType(node->Pos(), *node->GetTypeAnn()->Cast<TListExprType>()->GetItemType(), ctx);
+ const auto structure = node->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
const auto urlType = ExpandType(node->Pos(), *structure->GetItems()[structure->FindItem("Url").GetRef()]->GetItemType(), ctx);
const auto pathType = ExpandType(node->Pos(), *structure->GetItems()[structure->FindItem("Path").GetRef()]->GetItemType(), ctx);
@@ -3410,32 +3410,32 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
map["Collect"] = std::bind(&OptimizeCollect, _1, _2);
map["LazyList"] = std::bind(&DropDuplicate, _1, _2);
- map["FlatMap"] = std::bind(&SimpleFlatMap<false>, _1, _2, _3);
- map["OrderedFlatMap"] = std::bind(&SimpleFlatMap<true>, _1, _2, _3);
+ map["FlatMap"] = std::bind(&SimpleFlatMap<false>, _1, _2, _3);
+ map["OrderedFlatMap"] = std::bind(&SimpleFlatMap<true>, _1, _2, _3);
map["MultiMap"] = std::bind(&CanonizeMultiMap<false>, _1, _2);
map["OrderedMultiMap"] = std::bind(&CanonizeMultiMap<true>, _1, _2);
- map["LMap"] = map["OrderedLMap"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- if (CanRewriteToEmptyContainer(*node)) {
+ map["LMap"] = map["OrderedLMap"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ if (CanRewriteToEmptyContainer(*node)) {
const auto& inputToCheck = SkipCallables(node->Head(), SkippableCallables);
- if (IsEmptyContainer(inputToCheck) || IsEmpty(inputToCheck, *optCtx.Types)) {
- YQL_CLOG(DEBUG, Core) << "Empty " << node->Content() << " over " << inputToCheck.Content();
- auto res = ctx.NewCallable(inputToCheck.Pos(), GetEmptyCollectionName(node->GetTypeAnn()), {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
- return KeepConstraints(res, *node, ctx);
- }
-
+ if (IsEmptyContainer(inputToCheck) || IsEmpty(inputToCheck, *optCtx.Types)) {
+ YQL_CLOG(DEBUG, Core) << "Empty " << node->Content() << " over " << inputToCheck.Content();
+ auto res = ctx.NewCallable(inputToCheck.Pos(), GetEmptyCollectionName(node->GetTypeAnn()), {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
+ return KeepConstraints(res, *node, ctx);
+ }
+
const auto& lambdaRootToCheck = SkipCallables(node->Tail().Tail(), SkippableCallables);
- if (IsEmptyContainer(lambdaRootToCheck) || IsEmpty(lambdaRootToCheck, *optCtx.Types)) {
- YQL_CLOG(DEBUG, Core) << "Empty " << node->Content() << " with " << lambdaRootToCheck.Content();
- auto res = ctx.NewCallable(lambdaRootToCheck.Pos(), GetEmptyCollectionName(node->GetTypeAnn()), {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
- return KeepConstraints(res, *node, ctx);
- }
- }
-
- return node;
- };
-
+ if (IsEmptyContainer(lambdaRootToCheck) || IsEmpty(lambdaRootToCheck, *optCtx.Types)) {
+ YQL_CLOG(DEBUG, Core) << "Empty " << node->Content() << " with " << lambdaRootToCheck.Content();
+ auto res = ctx.NewCallable(lambdaRootToCheck.Pos(), GetEmptyCollectionName(node->GetTypeAnn()), {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
+ return KeepConstraints(res, *node, ctx);
+ }
+ }
+
+ return node;
+ };
+
map["FlatMapToEquiJoin"] = map["OrderedFlatMapToEquiJoin"] =
[](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) {
const auto self = TCoFlatMapToEquiJoinBase(node);
@@ -3471,7 +3471,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
}
YQL_CLOG(DEBUG, Core) << "FuseSkipNullMembers";
- return Build<TCoSkipNullMembers>(ctx, innerSkip.Pos())
+ return Build<TCoSkipNullMembers>(ctx, innerSkip.Pos())
.Input(innerSkip.Input())
.Members()
.Add(membersList)
@@ -3536,12 +3536,12 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
map["Map"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
YQL_CLOG(DEBUG, Core) << "Canonize " << node->Content();
- return ConvertMapToFlatmap<TCoMap, TCoFlatMap>(TCoMap(node), ctx);
+ return ConvertMapToFlatmap<TCoMap, TCoFlatMap>(TCoMap(node), ctx);
};
map["OrderedMap"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
YQL_CLOG(DEBUG, Core) << "Canonize " << node->Content();
- return ConvertMapToFlatmap<TCoOrderedMap, TCoOrderedFlatMap>(TCoOrderedMap(node), ctx);
+ return ConvertMapToFlatmap<TCoOrderedMap, TCoOrderedFlatMap>(TCoOrderedMap(node), ctx);
};
map["ExtractMembers"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
@@ -3555,7 +3555,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
return ctx.ChangeChild(*node, 0U, node->Head().HeadPtr());
}
- if (node->Head().IsCallable({"Nothing", "List"}) && 1U == node->Head().ChildrenSize()) {
+ if (node->Head().IsCallable({"Nothing", "List"}) && 1U == node->Head().ChildrenSize()) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
return ctx.ChangeChild(node->Head(), 0U, ExpandType(node->Pos(), *node->GetTypeAnn(), ctx));
}
@@ -3575,16 +3575,16 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
return ctx.ChangeChildren(node->Head(), std::move(args));
}
- if (node->Head().IsCallable("AssumeAllMembersNullableAtOnce")) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
- return ctx.SwapWithHead(*node);
- }
-
+ if (node->Head().IsCallable("AssumeAllMembersNullableAtOnce")) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
+ return ctx.SwapWithHead(*node);
+ }
+
return node;
};
map["Lookup"] = std::bind(&OptimizeContains<false, true>, _1, _2);
- map["Contains"] = std::bind(&OptimizeContains<false>, _1, _2);
+ map["Contains"] = std::bind(&OptimizeContains<false>, _1, _2);
map["ListHas"] = std::bind(&OptimizeContains<true>, _1, _2);
map["SqlIn"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext&) {
auto collection = node->HeadPtr();
@@ -3737,7 +3737,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
if (!value) {
YQL_CLOG(DEBUG, Core) << node->Content() << " with " << node->Tail().Content() << " '" << node->Tail().Head().Content();
auto res = ctx.NewCallable(node->Tail().Pos(), GetEmptyCollectionName(node->GetTypeAnn()), {ExpandType(node->Pos(), *node->GetTypeAnn(), ctx)});
- res = KeepConstraints(res, *node, ctx);
+ res = KeepConstraints(res, *node, ctx);
return KeepColumnOrder(res, *node, ctx, *optCtx.Types);
}
}
@@ -3751,26 +3751,26 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
map["TakeWhileInclusive"] = std::bind(&OptimizeWhile<true, true>, _1, _2);
map["SkipWhileInclusive"] = std::bind(&OptimizeWhile<false, true>, _1, _2);
- map[TCoExtend::CallableName()] = map[TCoOrderedExtend::CallableName()] = map[TCoMerge::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ map[TCoExtend::CallableName()] = map[TCoOrderedExtend::CallableName()] = map[TCoMerge::CallableName()] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
if (node->ChildrenSize() == 1) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over one child";
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over one child";
return node->HeadPtr();
}
for (ui32 i = 0; i < node->ChildrenSize(); ++i) {
auto& child = SkipCallables(*node->Child(i), SkippableCallables);
- if (IsEmptyContainer(child) || IsEmpty(child, *optCtx.Types)) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over empty list";
+ if (IsEmptyContainer(child) || IsEmpty(child, *optCtx.Types)) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over empty list";
if (node->ChildrenSize() == 2) {
- return KeepConstraints(node->ChildPtr(1 - i), *node, ctx);
+ return KeepConstraints(node->ChildPtr(1 - i), *node, ctx);
}
TExprNode::TListType newChildren = node->ChildrenList();
newChildren.erase(newChildren.begin() + i);
- return KeepConstraints(ctx.ChangeChildren(*node, std::move(newChildren)), *node, ctx);
+ return KeepConstraints(ctx.ChangeChildren(*node, std::move(newChildren)), *node, ctx);
}
- if (TCoExtendBase::Match(node->Child(i))) {
+ if (TCoExtendBase::Match(node->Child(i))) {
TExprNode::TListType newChildren = node->ChildrenList();
TExprNode::TListType insertedChildren = node->Child(i)->ChildrenList();
newChildren.erase(newChildren.begin() + i);
@@ -3781,7 +3781,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
for (ui32 i = 0; i < node->ChildrenSize() - 1; ++i) {
if (node->Child(i)->IsCallable("AsList") && node->Child(i + 1)->IsCallable("AsList")) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over 2 or more AsList";
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over 2 or more AsList";
ui32 j = i + 2;
for (; j < node->ChildrenSize(); ++j) {
if (!node->Child(j)->IsCallable("AsList")) {
@@ -3835,16 +3835,16 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
map["Length"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
const auto& nodeToCheck = SkipCallables(node->Head(), SkippableCallables);
- if (nodeToCheck.IsCallable("AsList")) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content();
+ if (nodeToCheck.IsCallable("AsList")) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content();
return ctx.NewCallable(node->Pos(), "Uint64",
- { ctx.NewAtom(node->Pos(), ToString(nodeToCheck.ChildrenSize()), TNodeFlags::Default) });
+ { ctx.NewAtom(node->Pos(), ToString(nodeToCheck.ChildrenSize()), TNodeFlags::Default) });
}
- if (nodeToCheck.IsCallable("List")) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content();
+ if (nodeToCheck.IsCallable("List")) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content();
return ctx.NewCallable(node->Pos(), "Uint64",
- { ctx.NewAtom(node->Pos(), ToString(nodeToCheck.ChildrenSize() - 1), TNodeFlags::Default) });
+ { ctx.NewAtom(node->Pos(), ToString(nodeToCheck.ChildrenSize() - 1), TNodeFlags::Default) });
}
if (IsListReorder(nodeToCheck) || nodeToCheck.IsCallable(
@@ -3898,13 +3898,13 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
const auto& nodeToCheck = SkipCallables(node->Head(), SkippableCallables);
if (nodeToCheck.IsCallable({"AsList","AsDict"})) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content();
- return MakeBool(node->Pos(), nodeToCheck.ChildrenSize() > 0U, ctx);
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content();
+ return MakeBool(node->Pos(), nodeToCheck.ChildrenSize() > 0U, ctx);
}
if (nodeToCheck.IsCallable({"List","Dict"})) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content();
- return MakeBool(node->Pos(), nodeToCheck.ChildrenSize() > 1U, ctx);
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over " << nodeToCheck.Content();
+ return MakeBool(node->Pos(), nodeToCheck.ChildrenSize() > 1U, ctx);
}
if (IsListReorder(nodeToCheck) || nodeToCheck.IsCallable(
@@ -4002,11 +4002,11 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
map["AddMember"] = std::bind(&ExpandAddMember, _1, _2);
map["ReplaceMember"] = std::bind(&ExpandReplaceMember, _1, _2);
- map["RemovePrefixMembers"] = std::bind(&ExpandRemovePrefixMembers, _1, _2);
+ map["RemovePrefixMembers"] = std::bind(&ExpandRemovePrefixMembers, _1, _2);
map["FlattenByColumns"] = std::bind(&ExpandFlattenByColumns, _1, _2);
- map["AsStruct"] = std::bind(&OptimizeAsStruct, _1, _2);
+ map["AsStruct"] = std::bind(&OptimizeAsStruct, _1, _2);
map["Nth"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
if (node->Head().Type() == TExprNode::List) {
@@ -4207,14 +4207,14 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
map["Fold"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
Y_UNUSED(ctx);
- if (node->Child(1)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct) {
+ if (node->Child(1)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct) {
const auto count = node->Child(1)->GetTypeAnn()->Cast<TStructExprType>()->GetSize();
if (count == 0) {
YQL_CLOG(DEBUG, Core) << node->Content() << " with empty struct as state";
return node->ChildPtr(1); // singleton
}
}
- else if (node->Child(1)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
+ else if (node->Child(1)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
const auto count = node->Child(1)->GetTypeAnn()->Cast<TTupleExprType>()->GetSize();
if (count == 0) {
YQL_CLOG(DEBUG, Core) << node->Content() << " with empty tuple as state";
@@ -4324,11 +4324,11 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
map["<"] = map["<="] = map[">"] = map[">="] = std::bind(&OptimizeCompare, _1, _2);;
- map["Sort"] = std::bind(&OptimizeReorder<false, true>, _1, _2);
- map["AssumeSorted"] = std::bind(&OptimizeReorder<false, false>, _1, _2);
+ map["Sort"] = std::bind(&OptimizeReorder<false, true>, _1, _2);
+ map["AssumeSorted"] = std::bind(&OptimizeReorder<false, false>, _1, _2);
- map["Top"] = std::bind(&OptimizeReorder<true, false>, _1, _2);
- map["TopSort"] = std::bind(&OptimizeReorder<true, true>, _1, _2);
+ map["Top"] = std::bind(&OptimizeReorder<true, false>, _1, _2);
+ map["TopSort"] = std::bind(&OptimizeReorder<true, true>, _1, _2);
map["Minus"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
if (node->Head().IsCallable("Minus")) {
@@ -4355,9 +4355,9 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
map["CastStruct"] = std::bind(&ExpandCastStruct, _1, _2);
- map["Append"] = std::bind(&OptimizeInsert<true>, _1, _2, _3);
- map["Insert"] = std::bind(&OptimizeInsert<true>, _1, _2, _3);
- map["Prepend"] = std::bind(&OptimizeInsert<false>, _1, _2, _3);
+ map["Append"] = std::bind(&OptimizeInsert<true>, _1, _2, _3);
+ map["Insert"] = std::bind(&OptimizeInsert<true>, _1, _2, _3);
+ map["Prepend"] = std::bind(&OptimizeInsert<false>, _1, _2, _3);
map["Extract"] = std::bind(&ExpandExtract<false>, _1, _2);
map["OrderedExtract"] = std::bind(&ExpandExtract<true>, _1, _2);
@@ -4366,7 +4366,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
map["UnionMerge"] = std::bind(&ExpandUnionAll<true>, _1, _2, _3);
map["Aggregate"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
- TCoAggregate self(node);
+ TCoAggregate self(node);
if (self.Keys().Size() == 0 && !HasPayload(self)) {
YQL_CLOG(DEBUG, Core) << node->Content() << " with empty fields";
return ctx.NewCallable(node->Pos(), "AsList", {ctx.NewCallable(node->Pos(), "AsStruct", {})});
@@ -4434,19 +4434,19 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
return node;
};
- map["EquiJoin"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- auto ret = HandleEmptyListInJoin(node, ctx, *optCtx.Types);
+ map["EquiJoin"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ auto ret = HandleEmptyListInJoin(node, ctx, *optCtx.Types);
if (ret != node) {
YQL_CLOG(DEBUG, Core) << "HandleEmptyListInJoin";
return ret;
}
- ret = HandleUniqueListInJoin(node, ctx, *optCtx.Types);
- if (ret != node) {
- YQL_CLOG(DEBUG, Core) << "HandleUniqueListInJoin";
- return ret;
- }
-
+ ret = HandleUniqueListInJoin(node, ctx, *optCtx.Types);
+ if (ret != node) {
+ YQL_CLOG(DEBUG, Core) << "HandleUniqueListInJoin";
+ return ret;
+ }
+
ui32 inputsCount = node->ChildrenSize() - 2;
for (ui32 i = 0; i < inputsCount; ++i) {
if (IsListReorder(node->Child(i)->Head())) {
@@ -4535,7 +4535,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
return ctx.NewCallable(node->Pos(), "Just", { node->Head().HeadPtr() });
} else {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content() << " - different index";
- return ctx.NewCallable(node->Pos(), "Nothing", { ExpandType(node->Pos(), *node->GetTypeAnn(), ctx) });
+ return ctx.NewCallable(node->Pos(), "Nothing", { ExpandType(node->Pos(), *node->GetTypeAnn(), ctx) });
}
}
@@ -4545,7 +4545,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
map["Way"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
if (node->Head().IsCallable("Nothing")) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
- return ctx.NewCallable(node->Pos(), "Nothing", { ExpandType(node->Pos(), *node->GetTypeAnn(), ctx) });
+ return ctx.NewCallable(node->Pos(), "Nothing", { ExpandType(node->Pos(), *node->GetTypeAnn(), ctx) });
}
if (node->Head().IsCallable("Just")) {
@@ -4617,54 +4617,54 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
}
}
- if (node->ChildrenSize() % 2 == 1) { // No default value
- bool allJust = true;
- TNodeSet uniqLambdas;
- for (ui32 index = 1; index < node->ChildrenSize(); index += 2) {
- uniqLambdas.insert(node->Child(index + 1));
- if (!TCoJust::Match(node->Child(index + 1)->Child(1))) {
- allJust = false;
- }
- }
-
- if (uniqLambdas.size() == 1 && node->ChildrenSize() > 3) {
+ if (node->ChildrenSize() % 2 == 1) { // No default value
+ bool allJust = true;
+ TNodeSet uniqLambdas;
+ for (ui32 index = 1; index < node->ChildrenSize(); index += 2) {
+ uniqLambdas.insert(node->Child(index + 1));
+ if (!TCoJust::Match(node->Child(index + 1)->Child(1))) {
+ allJust = false;
+ }
+ }
+
+ if (uniqLambdas.size() == 1 && node->ChildrenSize() > 3) {
YQL_CLOG(DEBUG, Core) << node->Content() << " - all equal lambdas";
- return ctx.Builder(node->Pos())
- .Apply(node->ChildPtr(2))
- .With(0)
- .Callable("VariantItem")
+ return ctx.Builder(node->Pos())
+ .Apply(node->ChildPtr(2))
+ .With(0)
+ .Callable("VariantItem")
.Add(0, node->HeadPtr())
- .Seal()
- .Done()
- .Seal()
- .Build();
- }
-
- if (allJust) {
+ .Seal()
+ .Done()
+ .Seal()
+ .Build();
+ }
+
+ if (allJust) {
YQL_CLOG(DEBUG, Core) << node->Content() << " - extract Just";
- return ctx.Builder(node->Pos())
- .Callable("Just")
- .Callable(0, "Visit")
+ return ctx.Builder(node->Pos())
+ .Callable("Just")
+ .Callable(0, "Visit")
.Add(0, node->HeadPtr())
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- for (ui32 i = 1; i < node->ChildrenSize(); i += 2) {
- parent.Add(i, node->ChildPtr(i));
- auto visitLambda = node->Child(i + 1);
- parent.Lambda(i + 1, visitLambda->Pos())
- .Param("item")
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ for (ui32 i = 1; i < node->ChildrenSize(); i += 2) {
+ parent.Add(i, node->ChildPtr(i));
+ auto visitLambda = node->Child(i + 1);
+ parent.Lambda(i + 1, visitLambda->Pos())
+ .Param("item")
.ApplyPartial(visitLambda->HeadPtr(), visitLambda->Tail().HeadPtr())
- .With(0, "item")
- .Seal()
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Seal()
- .Build();
- }
- }
-
+ .With(0, "item")
+ .Seal()
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Seal()
+ .Build();
+ }
+ }
+
return node;
};
@@ -4704,8 +4704,8 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
return node;
};
-
- map["Switch"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
+
+ map["Switch"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
TExprNode::TPtr flatMap;
for (auto i = 3U; !flatMap && i < node->ChildrenSize(); ++++i) {
flatMap = FindNode(node->Child(i)->TailPtr(),
@@ -4732,185 +4732,185 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
}
const auto inputItemType = GetSeqItemType(node->Head().GetTypeAnn());
- const bool singleInput = inputItemType->GetKind() != ETypeAnnotationKind::Variant;
-
- TDynBitMap usedIndicies;
- TExprNode::TListType lambdas;
- TExprNode::TListType indicies;
- TExprNode::TListType castStructs;
- ETypeAnnotationKind targetType = singleInput ? ETypeAnnotationKind::List : ETypeAnnotationKind::Optional;
- const bool singleHandler = node->ChildrenSize() == 4;
- bool ordered = false;
-
- if (singleInput && singleHandler && node->Child(2)->ChildrenSize() == 1) { // Exactly one index
+ const bool singleInput = inputItemType->GetKind() != ETypeAnnotationKind::Variant;
+
+ TDynBitMap usedIndicies;
+ TExprNode::TListType lambdas;
+ TExprNode::TListType indicies;
+ TExprNode::TListType castStructs;
+ ETypeAnnotationKind targetType = singleInput ? ETypeAnnotationKind::List : ETypeAnnotationKind::Optional;
+ const bool singleHandler = node->ChildrenSize() == 4;
+ bool ordered = false;
+
+ if (singleInput && singleHandler && node->Child(2)->ChildrenSize() == 1) { // Exactly one index
YQL_CLOG(DEBUG, Core) << node->Content() << " with single input and single handler";
- return ctx.Builder(node->Pos())
- .Apply(node->ChildPtr(3)) // handler lambda
+ return ctx.Builder(node->Pos())
+ .Apply(node->ChildPtr(3)) // handler lambda
.With(0, node->HeadPtr()) // Switch input
- .Seal()
- .Build();
- }
-
- for (ui32 i = 2; i < node->ChildrenSize(); i += 2) {
- if (node->Child(i)->ChildrenSize() != 1) {
- return node;
- }
- if (!singleInput) {
+ .Seal()
+ .Build();
+ }
+
+ for (ui32 i = 2; i < node->ChildrenSize(); i += 2) {
+ if (node->Child(i)->ChildrenSize() != 1) {
+ return node;
+ }
+ if (!singleInput) {
ui32 index = FromString<ui32>(node->Child(i)->Head().Content());
- if (usedIndicies.Test(index)) {
- return node;
- }
- usedIndicies.Set(index);
+ if (usedIndicies.Test(index)) {
+ return node;
+ }
+ usedIndicies.Set(index);
indicies.push_back(node->Child(i)->HeadPtr());
- }
-
- auto lambda = node->Child(i + 1);
+ }
+
+ auto lambda = node->Child(i + 1);
if (&lambda->Head().Head() == &lambda->Tail()) {
- // Trivial lambda
- ordered = ordered || lambda->GetConstraint<TSortedConstraintNode>();
- lambdas.emplace_back();
- castStructs.emplace_back();
- }
- else if (TCoFlatMapBase::Match(lambda->Child(1))) {
- ordered = ordered || TCoOrderedFlatMap::Match(lambda->Child(1)) || lambda->GetConstraint<TSortedConstraintNode>();
- auto flatMapInput = lambda->Child(1)->Child(0);
- const TTypeAnnotationNode* castType = nullptr;
- if (TCoExtractMembers::Match(flatMapInput)) {
+ // Trivial lambda
+ ordered = ordered || lambda->GetConstraint<TSortedConstraintNode>();
+ lambdas.emplace_back();
+ castStructs.emplace_back();
+ }
+ else if (TCoFlatMapBase::Match(lambda->Child(1))) {
+ ordered = ordered || TCoOrderedFlatMap::Match(lambda->Child(1)) || lambda->GetConstraint<TSortedConstraintNode>();
+ auto flatMapInput = lambda->Child(1)->Child(0);
+ const TTypeAnnotationNode* castType = nullptr;
+ if (TCoExtractMembers::Match(flatMapInput)) {
castType = GetSeqItemType(flatMapInput->GetTypeAnn());
- flatMapInput = flatMapInput->Child(0);
- }
-
+ flatMapInput = flatMapInput->Child(0);
+ }
+
if (flatMapInput != lambda->Head().Child(0)) { // FlatMap input == Switch lambda arg
- return node;
- }
-
- auto flatMapLambda = lambda->Child(1)->ChildPtr(1);
- switch (flatMapLambda->GetTypeAnn()->GetKind()) {
- case ETypeAnnotationKind::Optional:
- if (!singleHandler && flatMapLambda->GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Variant) {
- return node;
- }
- break;
- case ETypeAnnotationKind::List:
- if (!singleHandler && flatMapLambda->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Variant) {
- return node;
- }
- if (targetType != ETypeAnnotationKind::Stream) {
- targetType = ETypeAnnotationKind::List;
- }
- break;
- case ETypeAnnotationKind::Stream:
- if (!singleHandler && flatMapLambda->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Variant) {
- return node;
- }
- targetType = ETypeAnnotationKind::Stream;
- break;
+ return node;
+ }
+
+ auto flatMapLambda = lambda->Child(1)->ChildPtr(1);
+ switch (flatMapLambda->GetTypeAnn()->GetKind()) {
+ case ETypeAnnotationKind::Optional:
+ if (!singleHandler && flatMapLambda->GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Variant) {
+ return node;
+ }
+ break;
+ case ETypeAnnotationKind::List:
+ if (!singleHandler && flatMapLambda->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Variant) {
+ return node;
+ }
+ if (targetType != ETypeAnnotationKind::Stream) {
+ targetType = ETypeAnnotationKind::List;
+ }
+ break;
+ case ETypeAnnotationKind::Stream:
+ if (!singleHandler && flatMapLambda->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Variant) {
+ return node;
+ }
+ targetType = ETypeAnnotationKind::Stream;
+ break;
case ETypeAnnotationKind::Flow:
if (!singleHandler && flatMapLambda->GetTypeAnn()->Cast<TFlowExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Variant) {
return node;
}
targetType = ETypeAnnotationKind::Flow;
break;
- default:
- YQL_ENSURE(false, "Unsupported FlatMap lambda return type: " << flatMapLambda->GetTypeAnn()->GetKind());
- }
- lambdas.push_back(std::move(flatMapLambda));
- castStructs.push_back(castType ? ExpandType(flatMapInput->Pos(), *castType, ctx) : TExprNode::TPtr());
- }
- else {
- return node;
- }
- }
-
- const auto flatMapName = ordered ? TCoOrderedFlatMap::CallableName() : TCoFlatMap::CallableName();
- const auto mapName = ordered ? TCoOrderedMap::CallableName() : TCoMap::CallableName();
- if (indicies.size() == 1) {
+ default:
+ YQL_ENSURE(false, "Unsupported FlatMap lambda return type: " << flatMapLambda->GetTypeAnn()->GetKind());
+ }
+ lambdas.push_back(std::move(flatMapLambda));
+ castStructs.push_back(castType ? ExpandType(flatMapInput->Pos(), *castType, ctx) : TExprNode::TPtr());
+ }
+ else {
+ return node;
+ }
+ }
+
+ const auto flatMapName = ordered ? TCoOrderedFlatMap::CallableName() : TCoFlatMap::CallableName();
+ const auto mapName = ordered ? TCoOrderedMap::CallableName() : TCoMap::CallableName();
+ if (indicies.size() == 1) {
YQL_CLOG(DEBUG, Core) << node->Content() << " with single trivial or FlatMap lambda";
- if (lambdas.front()) {
- return ctx.Builder(node->Pos())
- .Callable(flatMapName)
- .Callable(0, flatMapName)
+ if (lambdas.front()) {
+ return ctx.Builder(node->Pos())
+ .Callable(flatMapName)
+ .Callable(0, flatMapName)
.Add(0, node->HeadPtr())
- .Lambda(1)
- .Param("item")
- .Callable("Guess")
- .Arg(0, "item")
- .Add(1, indicies.front())
- .Seal()
- .Seal()
- .Seal()
- .Lambda(1)
- .Param("varItem")
- .Apply(lambdas.front())
- .With(0)
- .Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& {
- if (castStructs.front()) {
- builder.Callable("CastStruct")
- .Arg(0, "varItem")
- .Add(1, castStructs.front())
- .Seal();
- } else {
- builder.Arg("varItem");
- }
- return builder;
- })
- .Done()
- .Seal()
- .Seal()
- .Seal()
- .Build();
- }
- else {
- return ctx.Builder(node->Pos())
- .Callable(flatMapName)
+ .Lambda(1)
+ .Param("item")
+ .Callable("Guess")
+ .Arg(0, "item")
+ .Add(1, indicies.front())
+ .Seal()
+ .Seal()
+ .Seal()
+ .Lambda(1)
+ .Param("varItem")
+ .Apply(lambdas.front())
+ .With(0)
+ .Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& {
+ if (castStructs.front()) {
+ builder.Callable("CastStruct")
+ .Arg(0, "varItem")
+ .Add(1, castStructs.front())
+ .Seal();
+ } else {
+ builder.Arg("varItem");
+ }
+ return builder;
+ })
+ .Done()
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+ else {
+ return ctx.Builder(node->Pos())
+ .Callable(flatMapName)
.Add(0, node->HeadPtr())
- .Lambda(1)
- .Param("item")
- .Callable("Guess")
- .Arg(0, "item")
- .Add(1, indicies.front())
- .Seal()
- .Seal()
- .Seal()
- .Build();
- }
- }
-
+ .Lambda(1)
+ .Param("item")
+ .Callable("Guess")
+ .Arg(0, "item")
+ .Add(1, indicies.front())
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+ }
+
const auto outVarType = ExpandType(node->Pos(), *GetSeqItemType(node->GetTypeAnn()), ctx);
-
- TExprNode::TListType updatedLambdas;
- for (size_t i = 0; i < lambdas.size(); ++i) {
- auto arg = ctx.NewArgument(node->Pos(), "varItem");
- TExprNode::TPtr body;
- if (lambdas[i]) {
- body = ctx.Builder(node->Pos())
- .Callable(mapName)
- .Apply(0, lambdas[i])
- .With(0)
- .Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& {
- if (castStructs[i]) {
- builder.Callable("CastStruct")
- .Add(0, arg)
- .Add(1, castStructs[i])
- .Seal();
- } else {
- builder.Arg(arg);
- }
- return builder;
- })
- .Done()
- .Seal()
- .Lambda(1)
- .Param("mapItem")
- .Callable("Variant")
- .Arg(0, "mapItem")
+
+ TExprNode::TListType updatedLambdas;
+ for (size_t i = 0; i < lambdas.size(); ++i) {
+ auto arg = ctx.NewArgument(node->Pos(), "varItem");
+ TExprNode::TPtr body;
+ if (lambdas[i]) {
+ body = ctx.Builder(node->Pos())
+ .Callable(mapName)
+ .Apply(0, lambdas[i])
+ .With(0)
+ .Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& {
+ if (castStructs[i]) {
+ builder.Callable("CastStruct")
+ .Add(0, arg)
+ .Add(1, castStructs[i])
+ .Seal();
+ } else {
+ builder.Arg(arg);
+ }
+ return builder;
+ })
+ .Done()
+ .Seal()
+ .Lambda(1)
+ .Param("mapItem")
+ .Callable("Variant")
+ .Arg(0, "mapItem")
.Atom(1, ToString(i), TNodeFlags::Default)
- .Add(2, outVarType)
- .Seal()
- .Seal()
- .Seal()
- .Build();
- if (lambdas[i]->GetTypeAnn()->GetKind() != targetType) {
+ .Add(2, outVarType)
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ if (lambdas[i]->GetTypeAnn()->GetKind() != targetType) {
switch (targetType) {
case ETypeAnnotationKind::Flow:
body = ctx.NewCallable(node->Pos(), "ToFlow", {std::move(body)});
@@ -4923,106 +4923,106 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
break;
default:
break;
- }
- }
- }
- else {
- body = ctx.Builder(node->Pos())
- .Callable("Variant")
- .Add(0, arg)
+ }
+ }
+ }
+ else {
+ body = ctx.Builder(node->Pos())
+ .Callable("Variant")
+ .Add(0, arg)
.Atom(1, ToString(i), TNodeFlags::Default)
- .Add(2, outVarType)
- .Seal()
- .Build();
- if (ETypeAnnotationKind::List == targetType) {
+ .Add(2, outVarType)
+ .Seal()
+ .Build();
+ if (ETypeAnnotationKind::List == targetType) {
body = ctx.NewCallable(node->Pos(), "AsList", {std::move(body)});
- }
- else {
+ }
+ else {
body = ctx.NewCallable(node->Pos(), "Just", {std::move(body)});
if (ETypeAnnotationKind::Flow == targetType) {
body = ctx.NewCallable(node->Pos(), "ToFlow", {std::move(body)});
}
else if (ETypeAnnotationKind::Stream == targetType) {
body = ctx.NewCallable(node->Pos(), "ToStream", {std::move(body)});
- }
- }
- }
+ }
+ }
+ }
updatedLambdas.push_back(ctx.NewLambda(node->Pos(), ctx.NewArguments(node->Pos(), {std::move(arg)}), std::move(body)));
- }
-
- if (singleInput) {
+ }
+
+ if (singleInput) {
YQL_CLOG(DEBUG, Core) << "Replicating " << node->Content() << " with trivial or FlatMap lambdas";
- return ctx.Builder(node->Pos())
- .Callable(flatMapName)
+ return ctx.Builder(node->Pos())
+ .Callable(flatMapName)
.Add(0, node->HeadPtr())
- .Lambda(1)
- .Param("item")
+ .Lambda(1)
+ .Param("item")
.Callable(ordered ? TCoOrderedExtend::CallableName() : TCoExtend::CallableName())
- .Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& {
- for (size_t i = 0; i < updatedLambdas.size(); ++i) {
+ .Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& {
+ for (size_t i = 0; i < updatedLambdas.size(); ++i) {
builder.Apply(i, *updatedLambdas[i])
- .With(0, "item")
- .Seal();
- }
- return builder;
- })
- .Seal()
- .Seal()
- .Seal()
- .Build();
- }
-
+ .With(0, "item")
+ .Seal();
+ }
+ return builder;
+ })
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+
const auto inputVarTupleType = inputItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>();
-
+
YQL_CLOG(DEBUG, Core) << node->Content() << " with trivial or FlatMap lambdas";
- return ctx.Builder(node->Pos())
- .Callable(flatMapName)
+ return ctx.Builder(node->Pos())
+ .Callable(flatMapName)
.Add(0, node->HeadPtr())
- .Lambda(1)
- .Param("item")
- .Callable("Visit")
- .Arg(0, "item")
- .Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& {
- for (size_t i = 0; i < indicies.size(); ++i) {
- builder.Add(i * 2 + 1, indicies[i]);
- builder.Add(i * 2 + 2, updatedLambdas[i]);
- }
- if (indicies.size() < inputVarTupleType->GetSize()) {
+ .Lambda(1)
+ .Param("item")
+ .Callable("Visit")
+ .Arg(0, "item")
+ .Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& {
+ for (size_t i = 0; i < indicies.size(); ++i) {
+ builder.Add(i * 2 + 1, indicies[i]);
+ builder.Add(i * 2 + 2, updatedLambdas[i]);
+ }
+ if (indicies.size() < inputVarTupleType->GetSize()) {
builder.Callable(indicies.size() * 2 + 1, GetEmptyCollectionName(targetType))
.Add(0, ExpandType(node->Pos(), *MakeSequenceType(targetType, *GetSeqItemType(node->GetTypeAnn()), ctx), ctx))
.Seal();
- }
- return builder;
- })
- .Seal()
- .Seal()
- .Seal()
- .Build();
- };
-
- map["VariantItem"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
+ }
+ return builder;
+ })
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ };
+
+ map["VariantItem"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
if (TCoJust::Match(&node->Head())) {
YQL_CLOG(DEBUG, Core) << "Move " << node->Content() << " over " << node->Head().Content();
return ctx.SwapWithHead(*node);
- }
+ }
if (TCoOptionalIf::Match(&node->Head())) {
YQL_CLOG(DEBUG, Core) << "Move " << node->Content() << " over " << node->Head().Content();
return ctx.ChangeChild(node->Head(), 1U, ctx.ChangeChild(*node, 0U, node->Head().TailPtr()));
- }
+ }
if (TCoVariant::Match(&node->Head())) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
return node->Head().HeadPtr();
- }
+ }
if (TCoNothing::Match(&node->Head())) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
- return ctx.Builder(node->Pos())
- .Callable("Nothing")
- .Add(0, ExpandType(node->Pos(), *node->GetTypeAnn(), ctx))
- .Seal()
- .Build();
- }
- return node;
- };
+ return ctx.Builder(node->Pos())
+ .Callable("Nothing")
+ .Add(0, ExpandType(node->Pos(), *node->GetTypeAnn(), ctx))
+ .Seal()
+ .Build();
+ }
+ return node;
+ };
map["Untag"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
Y_UNUSED(ctx);
@@ -5068,7 +5068,7 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
YQL_CLOG(DEBUG, Core) << "ToDict without payload over list literal";
return ctx.Builder(node->Pos())
.Callable("DictFromKeys")
- .Add(0, ExpandType(node->Pos(), *node->GetTypeAnn()->Cast<TDictExprType>()->GetKeyType(), ctx))
+ .Add(0, ExpandType(node->Pos(), *node->GetTypeAnn()->Cast<TDictExprType>()->GetKeyType(), ctx))
.List(1)
.Do([&](TExprNodeBuilder& builder) -> TExprNodeBuilder& {
for (ui32 i = 0; i < node->Head().ChildrenSize(); ++i) {
@@ -5180,79 +5180,79 @@ void RegisterCoSimpleCallables1(TCallableOptimizerMap& map) {
Y_UNREACHABLE();
};
-
+
map["Unordered"] = map["UnorderedSubquery"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
if (node->Head().IsCallable("AsList")) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
return node->HeadPtr();
- }
-
- if (node->Head().IsCallable("AssumeSorted")) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
- return ctx.ChangeChild(*node, 0, node->Head().HeadPtr());
- }
-
- return node;
- };
-
- map["Demux"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
+ }
+
+ if (node->Head().IsCallable("AssumeSorted")) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
+ return ctx.ChangeChild(*node, 0, node->Head().HeadPtr());
+ }
+
+ return node;
+ };
+
+ map["Demux"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
if (TCoExtendBase::Match(&node->Head())) {
YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
-
- TExprNode::TListType demuxChildren;
+
+ TExprNode::TListType demuxChildren;
std::transform(node->Head().Children().begin(), node->Head().Children().end(),
- std::back_inserter(demuxChildren),
- [&] (const TExprNode::TPtr& n) {
- return Build<TCoDemux>(ctx, n->Pos()).Input(n).Done().Ptr();
- }
- );
-
+ std::back_inserter(demuxChildren),
+ [&] (const TExprNode::TPtr& n) {
+ return Build<TCoDemux>(ctx, n->Pos()).Input(n).Done().Ptr();
+ }
+ );
+
auto variantType = node->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TVariantExprType>();
- if (variantType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
- TExprNode::TListType resChildren;
- for (size_t i = 0; i < variantType->GetUnderlyingType()->Cast<TTupleExprType>()->GetSize(); ++i) {
- const auto nthIndex = ctx.NewAtom(node->Pos(), ToString(i));
- TExprNode::TListType extendChildren;
- for (auto& demux: demuxChildren) {
- extendChildren.push_back(Build<TCoNth>(ctx, demux->Pos())
- .Tuple(demux)
- .Index(nthIndex)
- .Done().Ptr()
- );
- }
+ if (variantType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
+ TExprNode::TListType resChildren;
+ for (size_t i = 0; i < variantType->GetUnderlyingType()->Cast<TTupleExprType>()->GetSize(); ++i) {
+ const auto nthIndex = ctx.NewAtom(node->Pos(), ToString(i));
+ TExprNode::TListType extendChildren;
+ for (auto& demux: demuxChildren) {
+ extendChildren.push_back(Build<TCoNth>(ctx, demux->Pos())
+ .Tuple(demux)
+ .Index(nthIndex)
+ .Done().Ptr()
+ );
+ }
resChildren.push_back(ctx.NewCallable(node->Pos(), node->Head().Content(), std::move(extendChildren)));
- }
- return ctx.NewList(node->Pos(), std::move(resChildren));
- }
- else {
- TExprNode::TListType resChildren;
- for (auto structItem: variantType->GetUnderlyingType()->Cast<TStructExprType>()->GetItems()) {
- const auto memberName = ctx.NewAtom(node->Pos(), structItem->GetName());
- TExprNode::TListType extendChildren;
- for (auto& demux: demuxChildren) {
- extendChildren.push_back(Build<TCoMember>(ctx, demux->Pos())
- .Struct(demux)
- .Name(memberName)
- .Done().Ptr()
- );
- }
+ }
+ return ctx.NewList(node->Pos(), std::move(resChildren));
+ }
+ else {
+ TExprNode::TListType resChildren;
+ for (auto structItem: variantType->GetUnderlyingType()->Cast<TStructExprType>()->GetItems()) {
+ const auto memberName = ctx.NewAtom(node->Pos(), structItem->GetName());
+ TExprNode::TListType extendChildren;
+ for (auto& demux: demuxChildren) {
+ extendChildren.push_back(Build<TCoMember>(ctx, demux->Pos())
+ .Struct(demux)
+ .Name(memberName)
+ .Done().Ptr()
+ );
+ }
auto extend = ctx.NewCallable(node->Pos(), node->Head().Content(), std::move(extendChildren));
- resChildren.push_back(ctx.NewList(node->Pos(), {memberName, extend}));
- }
- return ctx.NewCallable(node->Pos(), TCoAsStruct::CallableName(), std::move(resChildren));
- }
- }
-
- if (TCoMux::Match(&node->Head())) {
- auto variantType = node->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TVariantExprType>();
- if (variantType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple && node->Head().Head().IsList()) {
- YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
- return node->Head().HeadPtr();
- }
- }
-
- return node;
- };
+ resChildren.push_back(ctx.NewList(node->Pos(), {memberName, extend}));
+ }
+ return ctx.NewCallable(node->Pos(), TCoAsStruct::CallableName(), std::move(resChildren));
+ }
+ }
+
+ if (TCoMux::Match(&node->Head())) {
+ auto variantType = node->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TVariantExprType>();
+ if (variantType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple && node->Head().Head().IsList()) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << " over " << node->Head().Content();
+ return node->Head().HeadPtr();
+ }
+ }
+
+ return node;
+ };
map["JsonValue"] = [](const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& /*optCtx*/) {
/*
diff --git a/ydb/library/yql/core/common_opt/yql_co_simple2.cpp b/ydb/library/yql/core/common_opt/yql_co_simple2.cpp
index 6d59989b6a..e45408ea03 100644
--- a/ydb/library/yql/core/common_opt/yql_co_simple2.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_simple2.cpp
@@ -438,26 +438,26 @@ TExprNode::TPtr CheckCompareSame(const TExprNode::TPtr& node, TExprContext& ctx)
return node;
}
-TExprNode::TPtr IfPresentSubsetFields(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
- if (3U == node->ChildrenSize() && TCoFilterNullMembers::Match(&node->Head())) {
- auto children = node->ChildrenList();
- const auto& lambda = *children[TCoIfPresent::idx_PresentHandler];
-
- YQL_ENSURE(optCtx.ParentsMap);
- TSet<TStringBuf> usedFields;
- if (HaveFieldsSubset(lambda.TailPtr(), lambda.Head().Head(), usedFields, *optCtx.ParentsMap)) {
- YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
- children[TCoIfPresent::idx_Optional] = FilterByFields(children[TCoIfPresent::idx_Optional]->Pos(), children[TCoIfPresent::idx_Optional], usedFields, ctx, false);
- children[TCoIfPresent::idx_PresentHandler] = ctx.DeepCopyLambda(*children[TCoIfPresent::idx_PresentHandler]);
- return ctx.ChangeChildren(*node, std::move(children));
- }
- }
-
- return node;
-}
-
+TExprNode::TPtr IfPresentSubsetFields(const TExprNode::TPtr& node, TExprContext& ctx, TOptimizeContext& optCtx) {
+ if (3U == node->ChildrenSize() && TCoFilterNullMembers::Match(&node->Head())) {
+ auto children = node->ChildrenList();
+ const auto& lambda = *children[TCoIfPresent::idx_PresentHandler];
+
+ YQL_ENSURE(optCtx.ParentsMap);
+ TSet<TStringBuf> usedFields;
+ if (HaveFieldsSubset(lambda.TailPtr(), lambda.Head().Head(), usedFields, *optCtx.ParentsMap)) {
+ YQL_CLOG(DEBUG, Core) << node->Content() << "SubsetFields";
+ children[TCoIfPresent::idx_Optional] = FilterByFields(children[TCoIfPresent::idx_Optional]->Pos(), children[TCoIfPresent::idx_Optional], usedFields, ctx, false);
+ children[TCoIfPresent::idx_PresentHandler] = ctx.DeepCopyLambda(*children[TCoIfPresent::idx_PresentHandler]);
+ return ctx.ChangeChildren(*node, std::move(children));
+ }
+ }
+
+ return node;
}
+}
+
void RegisterCoSimpleCallables2(TCallableOptimizerMap& map) {
using namespace std::placeholders;
@@ -479,15 +479,15 @@ void RegisterCoSimpleCallables2(TCallableOptimizerMap& map) {
map["AggrMin"] = map["AggrMax"] = map["Coalesce"] = std::bind(&DropAggrOverSame, _1);
- map["StartsWith"] = map["EndsWith"] = std::bind(&CheckCompareSame<true, false>, _1, _2);
+ map["StartsWith"] = map["EndsWith"] = std::bind(&CheckCompareSame<true, false>, _1, _2);
map["=="] = map["<="] = map[">="] = std::bind(&CheckCompareSame<true, false>, _1, _2);
map["!="] = map["<"] = map[">"] = std::bind(&CheckCompareSame<false, false>, _1, _2);
map["AggrEquals"] = map["AggrLessOrEqual"] = map["AggrGreaterOrEqual"] = std::bind(&CheckCompareSame<true, true>, _1, _2);
map["AggrNotEquals"] = map["AggrLess"] = map["AggrGreater"] = std::bind(&CheckCompareSame<false, true>, _1, _2);
-
- map["IfPresent"] = std::bind(&IfPresentSubsetFields, _1, _2, _3);
+
+ map["IfPresent"] = std::bind(&IfPresentSubsetFields, _1, _2, _3);
}
}
diff --git a/ydb/library/yql/core/common_opt/yql_co_transformer.cpp b/ydb/library/yql/core/common_opt/yql_co_transformer.cpp
index 8a98716eac..0d9cfa2961 100644
--- a/ydb/library/yql/core/common_opt/yql_co_transformer.cpp
+++ b/ydb/library/yql/core/common_opt/yql_co_transformer.cpp
@@ -20,9 +20,9 @@ namespace {
class TCommonOptTransformer final : public TSyncTransformerBase {
public:
TCommonOptTransformer(TTypeAnnotationContext* typeCtx, bool final)
- : TypeCtx(typeCtx)
- , Final(final)
- {}
+ : TypeCtx(typeCtx)
+ , Final(final)
+ {}
IGraphTransformer::TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final;
void Rewind() final;
@@ -31,9 +31,9 @@ private:
const TCallableOptimizerMap& callables, TProcessedNodesSet& processedNodes,
bool withParents);
- IGraphTransformer::TStatus DoTransform(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx,
- const TFinalizingOptimizerMap& callables);
-
+ IGraphTransformer::TStatus DoTransform(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx,
+ const TFinalizingOptimizerMap& callables);
+
private:
TProcessedNodesSet SimpleProcessedNodes[TCoCallableRules::SIMPLE_STEPS];
TProcessedNodesSet FlowProcessedNodes[TCoCallableRules::FLOW_STEPS];
@@ -52,7 +52,7 @@ TAutoPtr<IGraphTransformer> CreateCommonOptFinalTransformer(TTypeAnnotationConte
return new TCommonOptTransformer(typeCtx, true);
}
-IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) {
+IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) {
IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok;
output = std::move(input);
@@ -74,11 +74,11 @@ IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(TExprNode::TPtr in
}
}
- status = DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().Finalizers);
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
+ status = DoTransform(input = std::move(output), output, ctx, TCoCallableRules::Instance().Finalizers);
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
return status;
}
@@ -97,8 +97,8 @@ void TCommonOptTransformer::Rewind() {
IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(
const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx,
const TCallableOptimizerMap& callables,
- TProcessedNodesSet& processedNodes, bool withParents)
-{
+ TProcessedNodesSet& processedNodes, bool withParents)
+{
TOptimizeExprSettings settings(TypeCtx);
settings.ProcessedNodes = &processedNodes;
settings.CustomInstantTypeTransformer = TypeCtx->CustomInstantTypeTransformer.Get();
@@ -131,42 +131,42 @@ IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(
}, ctx, settings);
}
-IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx,
- const TFinalizingOptimizerMap& callables)
-{
- TParentsMap parentsMap;
- TOptimizeContext optCtx;
- optCtx.Types = TypeCtx;
- GatherParents(*input, parentsMap);
- optCtx.ParentsMap = &parentsMap;
-
- TNodeOnNodeOwnedMap toOptimize;
- VisitExpr(input,
- [&toOptimize](const TExprNode::TPtr&) {
- return toOptimize.empty();
- },
- [&callables, &toOptimize, &ctx, &optCtx](const TExprNode::TPtr& node) {
- if (toOptimize.empty()) {
+IGraphTransformer::TStatus TCommonOptTransformer::DoTransform(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx,
+ const TFinalizingOptimizerMap& callables)
+{
+ TParentsMap parentsMap;
+ TOptimizeContext optCtx;
+ optCtx.Types = TypeCtx;
+ GatherParents(*input, parentsMap);
+ optCtx.ParentsMap = &parentsMap;
+
+ TNodeOnNodeOwnedMap toOptimize;
+ VisitExpr(input,
+ [&toOptimize](const TExprNode::TPtr&) {
+ return toOptimize.empty();
+ },
+ [&callables, &toOptimize, &ctx, &optCtx](const TExprNode::TPtr& node) {
+ if (toOptimize.empty()) {
const auto rule = callables.find(node->Content());
if (callables.cend() != rule) {
(rule->second)(node, toOptimize, ctx, optCtx);
- }
- }
- return toOptimize.empty();
- }
- );
-
- if (!toOptimize.empty()) {
+ }
+ }
+ return toOptimize.empty();
+ }
+ );
+
+ if (!toOptimize.empty()) {
TOptimizeExprSettings settings(TypeCtx);
- settings.VisitTuples = true;
- settings.CustomInstantTypeTransformer = TypeCtx->CustomInstantTypeTransformer.Get();
- return RemapExpr(input, output, toOptimize, ctx, settings);
- }
-
- output = input;
- return IGraphTransformer::TStatus::Ok;
-}
-
+ settings.VisitTuples = true;
+ settings.CustomInstantTypeTransformer = TypeCtx->CustomInstantTypeTransformer.Get();
+ return RemapExpr(input, output, toOptimize, ctx, settings);
+ }
+
+ output = input;
+ return IGraphTransformer::TStatus::Ok;
+}
+
const TCoCallableRules& TCoCallableRules::Instance() {
return *Singleton<TCoCallableRules>();
}
@@ -176,7 +176,7 @@ TCoCallableRules::TCoCallableRules() {
RegisterCoSimpleCallables2(SimpleCallables[SIMPLE_STEP_2]);
RegisterCoFlowCallables1(FlowCallables[FLOW_STEP_1]);
RegisterCoFlowCallables2(FlowCallables[FLOW_STEP_2]);
- RegisterCoFinalizers(Finalizers);
+ RegisterCoFinalizers(Finalizers);
RegisterCoFinalCallables(FinalCallables);
}
diff --git a/ydb/library/yql/core/common_opt/yql_co_transformer.h b/ydb/library/yql/core/common_opt/yql_co_transformer.h
index a2e8449e1e..3cd9dcd48c 100644
--- a/ydb/library/yql/core/common_opt/yql_co_transformer.h
+++ b/ydb/library/yql/core/common_opt/yql_co_transformer.h
@@ -4,8 +4,8 @@
#include <ydb/library/yql/core/yql_type_annotation.h>
#include <ydb/library/yql/core/yql_graph_transformer.h>
-#include <util/generic/ptr.h>
-
+#include <util/generic/ptr.h>
+
namespace NYql {
TAutoPtr<IGraphTransformer> CreateCommonOptTransformer(TTypeAnnotationContext* typeCtx);
diff --git a/ydb/library/yql/core/expr_nodes/ya.make b/ydb/library/yql/core/expr_nodes/ya.make
index 5f919fa0e4..c9f5da8710 100644
--- a/ydb/library/yql/core/expr_nodes/ya.make
+++ b/ydb/library/yql/core/expr_nodes/ya.make
@@ -1,10 +1,10 @@
LIBRARY()
-OWNER(
+OWNER(
spuchin
- g:yql
+ g:yql
g:yql_ydb_core
-)
+)
SRCS(
yql_expr_nodes.h
diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h
index 01cc777118..3577ff6506 100644
--- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h
+++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.h
@@ -11,13 +11,13 @@ namespace NNodes {
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.decl.inl.h>
-class TCoAtom : public NGenerated::TCoAtomStub<TExprBase> {
+class TCoAtom : public NGenerated::TCoAtomStub<TExprBase> {
public:
- explicit TCoAtom(const TExprNode* node)
- : TCoAtomStub(node) {}
+ explicit TCoAtom(const TExprNode* node)
+ : TCoAtomStub(node) {}
- explicit TCoAtom(const TExprNode::TPtr& node)
- : TCoAtomStub(node) {}
+ explicit TCoAtom(const TExprNode::TPtr& node)
+ : TCoAtomStub(node) {}
operator TStringBuf() const {
return Value();
@@ -28,19 +28,19 @@ public:
}
};
-inline bool operator==(const TCoAtom& lhs, const TStringBuf& rhs) {
+inline bool operator==(const TCoAtom& lhs, const TStringBuf& rhs) {
return lhs.Value() == rhs;
}
-class TCoArguments : public NGenerated::TCoArgumentsStub<TExprBase> {
+class TCoArguments : public NGenerated::TCoArgumentsStub<TExprBase> {
public:
- explicit TCoArguments(const TExprNode* node)
- : TCoArgumentsStub(node) {}
+ explicit TCoArguments(const TExprNode* node)
+ : TCoArgumentsStub(node) {}
- explicit TCoArguments(const TExprNode::TPtr& node)
- : TCoArgumentsStub(node) {}
+ explicit TCoArguments(const TExprNode::TPtr& node)
+ : TCoArgumentsStub(node) {}
- TCoArgument Arg(size_t index) const { return TCoArgument(Ref().ChildPtr(index)); }
+ TCoArgument Arg(size_t index) const { return TCoArgument(Ref().ChildPtr(index)); }
size_t Size() const { return Ref().ChildrenSize(); }
TChildIterator<TCoArgument> begin() const { return TChildIterator<TCoArgument>(*this); }
@@ -50,38 +50,38 @@ public:
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.defs.inl.h>
template<typename TParent>
-class TNodeBuilder<TParent, TCoWorld> : public NGenerated::TCoWorldBuilder<TParent>
+class TNodeBuilder<TParent, TCoWorld> : public NGenerated::TCoWorldBuilder<TParent>
{
public:
TNodeBuilder(TExprContext& ctx, TPositionHandle pos,
- typename NGenerated::TCoWorldBuilder<TParent>::BuildFuncType buildFunc,
- typename NGenerated::TCoWorldBuilder<TParent>::GetArgFuncType getArgFunc)
- : NGenerated::TCoWorldBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {}
+ typename NGenerated::TCoWorldBuilder<TParent>::BuildFuncType buildFunc,
+ typename NGenerated::TCoWorldBuilder<TParent>::GetArgFuncType getArgFunc)
+ : NGenerated::TCoWorldBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {}
- TCoWorld DoBuild() {
+ TCoWorld DoBuild() {
auto node = this->Ctx.NewWorld(this->Pos);
- return TCoWorld(node);
+ return TCoWorld(node);
}
};
template<typename TParent>
-class TNodeBuilder<TParent, TCoArgument> : public NGenerated::TCoArgumentBuilder<TParent>
+class TNodeBuilder<TParent, TCoArgument> : public NGenerated::TCoArgumentBuilder<TParent>
{
public:
TNodeBuilder(TExprContext& ctx, TPositionHandle pos,
- typename NGenerated::TCoArgumentBuilder<TParent>::BuildFuncType buildFunc,
- typename NGenerated::TCoArgumentBuilder<TParent>::GetArgFuncType getArgFunc)
- : NGenerated::TCoArgumentBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {}
+ typename NGenerated::TCoArgumentBuilder<TParent>::BuildFuncType buildFunc,
+ typename NGenerated::TCoArgumentBuilder<TParent>::GetArgFuncType getArgFunc)
+ : NGenerated::TCoArgumentBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {}
- TNodeBuilder<TParent, TCoArgument>& Name(const TStringBuf& value) {
+ TNodeBuilder<TParent, TCoArgument>& Name(const TStringBuf& value) {
this->NameHolder = this->Ctx.AppendString(value);
return *this;
}
- TCoArgument DoBuild() {
+ TCoArgument DoBuild() {
YQL_ENSURE(!NameHolder.empty());
auto node = this->Ctx.NewArgument(this->Pos, NameHolder);
- return TCoArgument(node);
+ return TCoArgument(node);
}
private:
@@ -89,50 +89,50 @@ private:
};
template<typename TParent>
-class TNodeBuilder<TParent, TCoAtom> : public NGenerated::TCoAtomBuilder<TParent>
+class TNodeBuilder<TParent, TCoAtom> : public NGenerated::TCoAtomBuilder<TParent>
{
public:
TNodeBuilder(TExprContext& ctx, TPositionHandle pos,
- typename NGenerated::TCoAtomBuilder<TParent>::BuildFuncType buildFunc,
- typename NGenerated::TCoAtomBuilder<TParent>::GetArgFuncType getArgFunc)
- : NGenerated::TCoAtomBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {}
+ typename NGenerated::TCoAtomBuilder<TParent>::BuildFuncType buildFunc,
+ typename NGenerated::TCoAtomBuilder<TParent>::GetArgFuncType getArgFunc)
+ : NGenerated::TCoAtomBuilder<TParent>(ctx, pos, buildFunc, getArgFunc) {}
- TNodeBuilder<TParent, TCoAtom>& Value(const TStringBuf& value, ui32 flags = TNodeFlags::ArbitraryContent) {
+ TNodeBuilder<TParent, TCoAtom>& Value(const TStringBuf& value, ui32 flags = TNodeFlags::ArbitraryContent) {
this->ValueHolder = this->Ctx.AppendString(value);
- this->Flags = flags;
+ this->Flags = flags;
return *this;
}
- TCoAtom DoBuild() {
- auto node = this->Ctx.NewAtom(this->Pos, ValueHolder, this->Flags);
- return TCoAtom(node);
+ TCoAtom DoBuild() {
+ auto node = this->Ctx.NewAtom(this->Pos, ValueHolder, this->Flags);
+ return TCoAtom(node);
}
TParent& Build() {
- return this->NGenerated::TCoAtomBuilder<TParent>::Build();
+ return this->NGenerated::TCoAtomBuilder<TParent>::Build();
}
- TParent& Build(const TStringBuf& value, ui32 flags = TNodeFlags::ArbitraryContent) {
+ TParent& Build(const TStringBuf& value, ui32 flags = TNodeFlags::ArbitraryContent) {
this->ValueHolder = this->Ctx.AppendString(value);
- this->Flags = flags;
+ this->Flags = flags;
return this->Build();
}
TParent& Build(i64 value) {
- return Build(ToString(value), TNodeFlags::Default);
+ return Build(ToString(value), TNodeFlags::Default);
}
private:
TStringBuf ValueHolder;
- ui32 Flags = TNodeFlags::ArbitraryContent;
+ ui32 Flags = TNodeFlags::ArbitraryContent;
};
template<typename TParent>
-class TNodeBuilder<TParent, TCoLambda> : public NGenerated::TCoLambdaBuilder<TParent>
+class TNodeBuilder<TParent, TCoLambda> : public NGenerated::TCoLambdaBuilder<TParent>
{
public:
TNodeBuilder(TExprContext& ctx, TPositionHandle pos,
- typename NGenerated::TCoLambdaBuilder<TParent>::BuildFuncType buildFunc,
+ typename NGenerated::TCoLambdaBuilder<TParent>::BuildFuncType buildFunc,
typename NGenerated::TCoLambdaBuilder<TParent>::GetArgFuncType getArgFunc,
std::shared_ptr<TMap<TStringBuf, TExprBase>> argsStore = std::make_shared<TMap<TStringBuf, TExprBase>>())
: NGenerated::TCoLambdaBuilder<TParent>(ctx, pos, buildFunc, [argsStore, getArgFunc] (const TStringBuf& argName) {
@@ -145,23 +145,23 @@ public:
, ArgsMap(argsStore)
{}
- TNodeBuilder<TParent, TCoLambda>& Args(const TCoArgument& node) {
+ TNodeBuilder<TParent, TCoLambda>& Args(const TCoArgument& node) {
Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid());
auto argsNode = this->Ctx.NewArguments(this->Pos, { node.Ptr() });
- this->ArgsHolder = TCoArguments(argsNode);
+ this->ArgsHolder = TCoArguments(argsNode);
return *this;
}
- TNodeBuilder<TParent, TCoLambda>& Args(const TCoArguments& node) {
+ TNodeBuilder<TParent, TCoLambda>& Args(const TCoArguments& node) {
Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid());
this->ArgsHolder = node;
return *this;
}
- TNodeBuilder<TParent, TCoLambda>& Args(std::initializer_list<TStringBuf> list)
+ TNodeBuilder<TParent, TCoLambda>& Args(std::initializer_list<TStringBuf> list)
{
Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid());
@@ -175,11 +175,11 @@ public:
}
auto argsNode = this->Ctx.NewArguments(this->Pos, std::move(argNodes));
- this->ArgsHolder = TCoArguments(argsNode);
+ this->ArgsHolder = TCoArguments(argsNode);
return *this;
}
- TNodeBuilder<TParent, TCoLambda>& Args(const std::vector<TCoArgument>& list)
+ TNodeBuilder<TParent, TCoLambda>& Args(const std::vector<TCoArgument>& list)
{
Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid());
@@ -189,23 +189,23 @@ public:
}
auto argsNode = this->Ctx.NewArguments(this->Pos, std::move(argNodes));
- this->ArgsHolder = TCoArguments(argsNode);
+ this->ArgsHolder = TCoArguments(argsNode);
return *this;
}
- TNodeBuilder<TParent, TCoLambda>& Args(const TExprNode::TListType& list)
- {
- Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid());
-
- auto argsNode = this->Ctx.NewArguments(this->Pos, TExprNode::TListType(list));
- this->ArgsHolder = TCoArguments(argsNode);
- return *this;
- }
-
- TCoLambda DoBuild() {
+ TNodeBuilder<TParent, TCoLambda>& Args(const TExprNode::TListType& list)
+ {
+ Y_VERIFY_DEBUG(!this->ArgsHolder.IsValid());
+
+ auto argsNode = this->Ctx.NewArguments(this->Pos, TExprNode::TListType(list));
+ this->ArgsHolder = TCoArguments(argsNode);
+ return *this;
+ }
+
+ TCoLambda DoBuild() {
auto node = this->Ctx.NewLambda(this->Pos, this->ArgsHolder.Cast().Ptr(), this->BodyHolder.Cast().Ptr());
- return TCoLambda(node);
+ return TCoLambda(node);
}
private:
@@ -259,12 +259,12 @@ public:
return BuildFunc(TExprApplier(Body.Cast().Ptr()));
}
- typename TParent::ResultType Done() {
- TParent& parent = Build();
- return parent.Value();
- }
-
- TNodeBuilder<TParent, TExprApplier>& Apply(TCoLambda lambda) {
+ typename TParent::ResultType Done() {
+ TParent& parent = Build();
+ return parent.Value();
+ }
+
+ TNodeBuilder<TParent, TExprApplier>& Apply(TCoLambda lambda) {
YQL_ENSURE(!Body);
Body = lambda.Body();
@@ -328,19 +328,19 @@ public:
buildFunc, GetArgFunc);
}
- template<typename TNode>
- TNodeBuilder<TNodeBuilder<TParent, TExprApplier>, TNode> With(TExprBase from) {
- YQL_ENSURE(Body);
-
- auto buildFunc = [from, this] (const TNode& node) mutable -> TNodeBuilder<TParent, TExprApplier>& {
- DoApply(from, node);
- return *this;
- };
-
- return TNodeBuilder<TNodeBuilder<TParent, TExprApplier>, TNode>(this->Ctx, this->Pos,
- buildFunc, GetArgFunc);
- }
-
+ template<typename TNode>
+ TNodeBuilder<TNodeBuilder<TParent, TExprApplier>, TNode> With(TExprBase from) {
+ YQL_ENSURE(Body);
+
+ auto buildFunc = [from, this] (const TNode& node) mutable -> TNodeBuilder<TParent, TExprApplier>& {
+ DoApply(from, node);
+ return *this;
+ };
+
+ return TNodeBuilder<TNodeBuilder<TParent, TExprApplier>, TNode>(this->Ctx, this->Pos,
+ buildFunc, GetArgFunc);
+ }
+
private:
void DoApply(TExprBase applyFrom, TExprBase applyTo) {
TExprNodeBuilder builder(this->Pos, this->Ctx, [this] (const TStringBuf& argName) {
@@ -359,7 +359,7 @@ private:
BuildFuncType BuildFunc;
TMaybeNode<TExprBase> Body;
- TMaybeNode<TCoArguments> Args;
+ TMaybeNode<TCoArguments> Args;
};
} // namespace NNodes
diff --git a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
index edfcaa1530..4047530a63 100644
--- a/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
+++ b/ydb/library/yql/core/expr_nodes/yql_expr_nodes.json
@@ -10,7 +10,7 @@
"ListBase": "TExprBase"
},
{
- "Name": "TCoAtom",
+ "Name": "TCoAtom",
"Base": "TExprBase",
"Definition": "Custom",
"Match": {"Type": "NodeType", "TypeName": "Atom"},
@@ -18,16 +18,16 @@
"Builder": {"Generate": "Custom"}
},
{
- "Name": "TCoAtomList",
- "ListBase": "TCoAtom"
+ "Name": "TCoAtomList",
+ "ListBase": "TCoAtom"
},
{
- "Name": "TCoNameValueTuple",
+ "Name": "TCoNameValueTuple",
"Base": "TExprBase",
"Match": {"Type": "Tuple"},
"Children": [
- {"Index": 0, "Name": "Name", "Type": "TCoAtom"},
- {"Index": 1, "Name": "Value", "Type": "TExprBase", "Optional": true}
+ {"Index": 0, "Name": "Name", "Type": "TCoAtom"},
+ {"Index": 1, "Name": "Value", "Type": "TExprBase", "Optional": true}
]
},
{
@@ -46,8 +46,8 @@
"ListBase": "TCoIndex"
},
{
- "Name": "TCoNameValueTupleList",
- "ListBase": "TCoNameValueTuple"
+ "Name": "TCoNameValueTupleList",
+ "ListBase": "TCoNameValueTuple"
},
{
"Name": "TCoChangefeed",
@@ -64,46 +64,46 @@
"ListBase": "TCoChangefeed"
},
{
- "Name": "TCoWorld",
+ "Name": "TCoWorld",
"Base": "TExprBase",
"Match": {"Type": "NodeType", "TypeName": "World"},
"Builder": {"Generate": "Custom"}
},
{
- "Name": "TCoArgument",
+ "Name": "TCoArgument",
"Base": "TExprBase",
"Match": {"Type": "NodeType", "TypeName": "Argument"},
"Content": {"Name": "Name", "Type": "TStringBuf"},
"Builder": {"Generate": "Custom"}
},
{
- "Name": "TCoArguments",
+ "Name": "TCoArguments",
"Base": "TExprBase",
"Definition": "Custom",
"Match": {"Type": "NodeType", "TypeName": "Arguments"},
"Builder": {"Generate": "None"}
},
{
- "Name": "TCoLambda",
+ "Name": "TCoLambda",
"Base": "TExprBase",
"Match": {"Type": "NodeType", "TypeName": "Lambda"},
"Builder": {"Generate": "Custom"},
"Children": [
- {"Index": 0, "Name": "Args", "Type": "TCoArguments"},
+ {"Index": 0, "Name": "Args", "Type": "TCoArguments"},
{"Index": 1, "Name": "Body", "Type": "TExprBase"}
]
},
{
- "Name": "TCoInputBase",
- "Base": "TCallable",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
- "Children": [
- {"Index": 0, "Name": "Input", "Type": "TExprBase"}
- ]
- },
- {
- "Name": "TCoApply",
+ "Name": "TCoInputBase",
+ "Base": "TCallable",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
+ "Children": [
+ {"Index": 0, "Name": "Input", "Type": "TExprBase"}
+ ]
+ },
+ {
+ "Name": "TCoApply",
"Base": "TFreeArgCallable",
"Match": {"Type": "Callable", "Name": "Apply"},
"Children": [
@@ -111,26 +111,26 @@
]
},
{
- "Name": "TCoRight",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "Right!"}
- },
- {
- "Name": "TCoLeft",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "Left!"}
+ "Name": "TCoRight",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "Right!"}
},
{
- "Name": "TCoCons",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "Cons!"},
- "Children": [
- {"Index": 0, "Name": "World", "Type": "TExprBase"},
- {"Index": 1, "Name": "Input", "Type": "TExprBase"}
- ]
+ "Name": "TCoLeft",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "Left!"}
},
{
- "Name": "TCoTypeOf",
+ "Name": "TCoCons",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "Cons!"},
+ "Children": [
+ {"Index": 0, "Name": "World", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Input", "Type": "TExprBase"}
+ ]
+ },
+ {
+ "Name": "TCoTypeOf",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "TypeOf"},
"Children": [
@@ -147,7 +147,7 @@
]
},
{
- "Name": "TCoNothing",
+ "Name": "TCoNothing",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Nothing"},
"Children": [
@@ -155,44 +155,44 @@
]
},
{
- "Name": "TCoVoid",
+ "Name": "TCoVoid",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Void"}
},
{
- "Name": "TCoNull",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "Null"}
- },
- {
- "Name": "TCoMapBase",
- "Base": "TCoInputBase",
- "Match": {"Type": "CallableBase"},
+ "Name": "TCoNull",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "Null"}
+ },
+ {
+ "Name": "TCoMapBase",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "CallableBase"},
"Children": [
- {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"}
+ {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"}
]
},
{
- "Name": "TCoMap",
- "Base": "TCoMapBase",
- "Match": {"Type": "Callable", "Name": "Map"}
- },
- {
- "Name": "TCoOrderedMap",
- "Base": "TCoMapBase",
- "Match": {"Type": "Callable", "Name": "OrderedMap"}
- },
- {
+ "Name": "TCoMap",
+ "Base": "TCoMapBase",
+ "Match": {"Type": "Callable", "Name": "Map"}
+ },
+ {
+ "Name": "TCoOrderedMap",
+ "Base": "TCoMapBase",
+ "Match": {"Type": "Callable", "Name": "OrderedMap"}
+ },
+ {
"Name": "TCoMultiMap",
"Base": "TCoMapBase",
"Match": {"Type": "Callable", "Name": "MultiMap"}
},
{
- "Name": "TCoLMap",
- "Base": "TCoMapBase",
- "Match": {"Type": "Callable", "Name": "LMap"}
- },
- {
+ "Name": "TCoLMap",
+ "Base": "TCoMapBase",
+ "Match": {"Type": "Callable", "Name": "LMap"}
+ },
+ {
"Name": "TCoOrderedLMap",
"Base": "TCoMapBase",
"Match": {"Type": "Callable", "Name": "OrderedLMap"}
@@ -218,47 +218,47 @@
"Match": {"Type": "Callable", "Name": "WideMap"}
},
{
- "Name": "TCoFilterBase",
- "Base": "TCoInputBase",
- "Match": {"Type": "CallableBase"},
+ "Name": "TCoFilterBase",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "CallableBase"},
"Children": [
{"Index": 1, "Name": "Lambda", "Type": "TCoLambda"},
{"Index": 2, "Name": "Limit", "Type": "TExprBase", "Optional": true}
]
},
{
- "Name": "TCoFilter",
- "Base": "TCoFilterBase",
- "Match": {"Type": "Callable", "Name": "Filter"}
- },
- {
- "Name": "TCoOrderedFilter",
- "Base": "TCoFilterBase",
- "Match": {"Type": "Callable", "Name": "OrderedFilter"}
- },
- {
+ "Name": "TCoFilter",
+ "Base": "TCoFilterBase",
+ "Match": {"Type": "Callable", "Name": "Filter"}
+ },
+ {
+ "Name": "TCoOrderedFilter",
+ "Base": "TCoFilterBase",
+ "Match": {"Type": "Callable", "Name": "OrderedFilter"}
+ },
+ {
"Name": "TCoWideFilter",
"Base": "TCoFilterBase",
"Match": {"Type": "Callable", "Name": "WideFilter"}
},
{
- "Name": "TCoFilterNullMembersBase",
- "Base": "TCoInputBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
+ "Name": "TCoFilterNullMembersBase",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
"Children": [
{"Index": 1, "Name": "Members", "Type": "TCoAtomList", "Optional": true}
]
},
{
- "Name": "TCoFilterNullMembers",
- "Base": "TCoFilterNullMembersBase",
- "Match": {"Type": "Callable", "Name": "FilterNullMembers"}
- },
- {
- "Name": "TCoSkipNullMembers",
- "Base": "TCoFilterNullMembersBase",
- "Match": {"Type": "Callable", "Name": "SkipNullMembers"}
+ "Name": "TCoFilterNullMembers",
+ "Base": "TCoFilterNullMembersBase",
+ "Match": {"Type": "Callable", "Name": "FilterNullMembers"}
+ },
+ {
+ "Name": "TCoSkipNullMembers",
+ "Base": "TCoFilterNullMembersBase",
+ "Match": {"Type": "Callable", "Name": "SkipNullMembers"}
},
{
"Name": "TCoFilterNullElementsBase",
@@ -280,44 +280,44 @@
"Match": {"Type": "Callable", "Name": "SkipNullElements"}
},
{
- "Name": "TCoRemoveSystemMembers",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "RemoveSystemMembers"}
- },
- {
- "Name": "TCoRemovePrefixMembers",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "RemovePrefixMembers"},
- "Children": [
- {"Index": 1, "Name": "Prefixes", "Type": "TCoAtomList"}
- ]
- },
- {
- "Name": "TCoSqlExtractKey",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "SqlExtractKey"},
+ "Name": "TCoRemoveSystemMembers",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "RemoveSystemMembers"}
+ },
+ {
+ "Name": "TCoRemovePrefixMembers",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "RemovePrefixMembers"},
+ "Children": [
+ {"Index": 1, "Name": "Prefixes", "Type": "TCoAtomList"}
+ ]
+ },
+ {
+ "Name": "TCoSqlExtractKey",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "SqlExtractKey"},
+ "Children": [
+ {"Index": 0, "Name": "Item", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Extractor", "Type": "TCoLambda"}
+ ]
+ },
+ {
+ "Name": "TCoFlatMapBase",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "CallableBase"},
"Children": [
- {"Index": 0, "Name": "Item", "Type": "TExprBase"},
- {"Index": 1, "Name": "Extractor", "Type": "TCoLambda"}
+ {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"}
]
},
{
- "Name": "TCoFlatMapBase",
- "Base": "TCoInputBase",
- "Match": {"Type": "CallableBase"},
- "Children": [
- {"Index": 1, "Name": "Lambda", "Type": "TCoLambda"}
- ]
+ "Name": "TCoFlatMap",
+ "Base": "TCoFlatMapBase",
+ "Match": {"Type": "Callable", "Name": "FlatMap"}
},
{
- "Name": "TCoFlatMap",
- "Base": "TCoFlatMapBase",
- "Match": {"Type": "Callable", "Name": "FlatMap"}
- },
- {
- "Name": "TCoOrderedFlatMap",
- "Base": "TCoFlatMapBase",
- "Match": {"Type": "Callable", "Name": "OrderedFlatMap"}
+ "Name": "TCoOrderedFlatMap",
+ "Base": "TCoFlatMapBase",
+ "Match": {"Type": "Callable", "Name": "OrderedFlatMap"}
},
{
"Name": "TCoFlatMapToEquiJoinBase",
@@ -338,11 +338,11 @@
"Match": {"Type": "Callable", "Name": "OrderedFlatMapToEquiJoin"}
},
{
- "Name": "TCoCollect",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "Collect"}
- },
- {
+ "Name": "TCoCollect",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "Collect"}
+ },
+ {
"Name": "TCoHead",
"Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "Head"}
@@ -358,17 +358,17 @@
"Match": {"Type": "CallableBase"}
},
{
- "Name": "TCoUnordered",
+ "Name": "TCoUnordered",
"Base": "TCoUnorderedBase",
- "Match": {"Type": "Callable", "Name": "Unordered"}
- },
- {
+ "Match": {"Type": "Callable", "Name": "Unordered"}
+ },
+ {
"Name": "TCoUnorderedSubquery",
"Base": "TCoUnorderedBase",
"Match": {"Type": "Callable", "Name": "UnorderedSubquery"}
},
{
- "Name": "TCoConditionalValueBase",
+ "Name": "TCoConditionalValueBase",
"Base": "TCallable",
"Match": {"Type": "CallableBase"},
"Children": [
@@ -377,78 +377,78 @@
]
},
{
- "Name": "TCoOptionalIf",
- "Base": "TCoConditionalValueBase",
+ "Name": "TCoOptionalIf",
+ "Base": "TCoConditionalValueBase",
"Match": {"Type": "Callable", "Name": "OptionalIf"}
},
{
- "Name": "TCoFlatOptionalIf",
- "Base": "TCoConditionalValueBase",
+ "Name": "TCoFlatOptionalIf",
+ "Base": "TCoConditionalValueBase",
"Match": {"Type": "Callable", "Name": "FlatOptionalIf"}
},
{
- "Name": "TCoListIf",
- "Base": "TCoConditionalValueBase",
+ "Name": "TCoListIf",
+ "Base": "TCoConditionalValueBase",
"Match": {"Type": "Callable", "Name": "ListIf"}
},
{
- "Name": "TCoFlatListIf",
- "Base": "TCoConditionalValueBase",
+ "Name": "TCoFlatListIf",
+ "Base": "TCoConditionalValueBase",
"Match": {"Type": "Callable", "Name": "FlatListIf"}
},
{
- "Name": "TCoJust",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "Just"}
+ "Name": "TCoJust",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "Just"}
},
{
- "Name": "TCoCombineByKey",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "CombineByKey"},
+ "Name": "TCoCombineByKey",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "CombineByKey"},
"Children": [
- {"Index": 1, "Name": "PreMapLambda", "Type": "TCoLambda"},
- {"Index": 2, "Name": "KeySelectorLambda", "Type": "TCoLambda"},
- {"Index": 3, "Name": "InitHandlerLambda", "Type": "TCoLambda"},
- {"Index": 4, "Name": "UpdateHandlerLambda", "Type": "TCoLambda"},
- {"Index": 5, "Name": "FinishHandlerLambda", "Type": "TCoLambda"}
+ {"Index": 1, "Name": "PreMapLambda", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "KeySelectorLambda", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "InitHandlerLambda", "Type": "TCoLambda"},
+ {"Index": 4, "Name": "UpdateHandlerLambda", "Type": "TCoLambda"},
+ {"Index": 5, "Name": "FinishHandlerLambda", "Type": "TCoLambda"}
]
},
{
- "Name": "TCoMin",
- "VarArgBase": "TExprBase",
+ "Name": "TCoMin",
+ "VarArgBase": "TExprBase",
"Match": {"Type": "Callable", "Name": "Min"}
},
{
- "Name": "TCoMax",
- "VarArgBase": "TExprBase",
- "Match": {"Type": "Callable", "Name": "Max"}
- },
- {
- "Name": "TCoCountBase",
- "Base": "TCoInputBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
+ "Name": "TCoMax",
+ "VarArgBase": "TExprBase",
+ "Match": {"Type": "Callable", "Name": "Max"}
+ },
+ {
+ "Name": "TCoCountBase",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
"Children": [
{"Index": 1, "Name": "Count", "Type": "TExprBase"}
]
},
{
- "Name": "TCoTake",
- "Base": "TCoCountBase",
- "Match": {"Type": "Callable", "Name": "Take"}
+ "Name": "TCoTake",
+ "Base": "TCoCountBase",
+ "Match": {"Type": "Callable", "Name": "Take"}
+ },
+ {
+ "Name": "TCoLimit",
+ "Base": "TCoCountBase",
+ "Match": {"Type": "Callable", "Name": "Limit"}
},
{
- "Name": "TCoLimit",
- "Base": "TCoCountBase",
- "Match": {"Type": "Callable", "Name": "Limit"}
+ "Name": "TCoSkip",
+ "Base": "TCoCountBase",
+ "Match": {"Type": "Callable", "Name": "Skip"}
},
{
- "Name": "TCoSkip",
- "Base": "TCoCountBase",
- "Match": {"Type": "Callable", "Name": "Skip"}
- },
- {
- "Name": "TCoAppend",
+ "Name": "TCoAppend",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Append"},
"Children": [
@@ -457,42 +457,42 @@
]
},
{
- "Name": "TCoExtendBase",
- "VarArgBase": "TExprBase",
- "Match": {"Type": "CallableBase"}
- },
- {
- "Name": "TCoExtend",
- "Base": "TCoExtendBase",
+ "Name": "TCoExtendBase",
+ "VarArgBase": "TExprBase",
+ "Match": {"Type": "CallableBase"}
+ },
+ {
+ "Name": "TCoExtend",
+ "Base": "TCoExtendBase",
"Match": {"Type": "Callable", "Name": "Extend"}
},
{
- "Name": "TCoOrderedExtend",
- "Base": "TCoExtendBase",
- "Match": {"Type": "Callable", "Name": "OrderedExtend"}
- },
- {
- "Name": "TCoMerge",
- "Base": "TCoExtendBase",
- "Match": {"Type": "Callable", "Name": "Merge"}
- },
- {
- "Name": "TCoUnionBase",
- "VarArgBase": "TExprBase",
- "Match": {"Type": "CallableBase"}
- },
- {
- "Name": "TCoUnionAll",
- "VarArgBase": "TCoUnionBase",
- "Match": {"Type": "Callable", "Name": "UnionAll"}
- },
- {
- "Name": "TCoUnionMerge",
- "VarArgBase": "TCoUnionBase",
- "Match": {"Type": "Callable", "Name": "UnionMerge"}
- },
- {
- "Name": "TCoHasItems",
+ "Name": "TCoOrderedExtend",
+ "Base": "TCoExtendBase",
+ "Match": {"Type": "Callable", "Name": "OrderedExtend"}
+ },
+ {
+ "Name": "TCoMerge",
+ "Base": "TCoExtendBase",
+ "Match": {"Type": "Callable", "Name": "Merge"}
+ },
+ {
+ "Name": "TCoUnionBase",
+ "VarArgBase": "TExprBase",
+ "Match": {"Type": "CallableBase"}
+ },
+ {
+ "Name": "TCoUnionAll",
+ "VarArgBase": "TCoUnionBase",
+ "Match": {"Type": "Callable", "Name": "UnionAll"}
+ },
+ {
+ "Name": "TCoUnionMerge",
+ "VarArgBase": "TCoUnionBase",
+ "Match": {"Type": "Callable", "Name": "UnionMerge"}
+ },
+ {
+ "Name": "TCoHasItems",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "HasItems"},
"Children": [
@@ -500,42 +500,42 @@
]
},
{
- "Name": "TCoLength",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "Length"},
- "Children": [
- {"Index": 0, "Name": "List", "Type": "TExprBase"}
- ]
- },
- {
- "Name": "TCoGroupByKey",
- "Base": "TCoInputBase",
+ "Name": "TCoLength",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "Length"},
+ "Children": [
+ {"Index": 0, "Name": "List", "Type": "TExprBase"}
+ ]
+ },
+ {
+ "Name": "TCoGroupByKey",
+ "Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "GroupByKey"},
"Children": [
- {"Index": 1, "Name": "KeySelectorLambda", "Type": "TCoLambda"},
- {"Index": 2, "Name": "HandlerLambda", "Type": "TCoLambda"}
- ]
- },
- {
- "Name": "TCoPartitionByKeyBase",
- "Base": "TCoInputBase",
- "Match": {"Type": "CallableBase"},
- "Children": [
- {"Index": 1, "Name": "KeySelectorLambda", "Type": "TCoLambda"},
- {"Index": 2, "Name": "SortDirections", "Type": "TExprBase"},
- {"Index": 3, "Name": "SortKeySelectorLambda", "Type": "TExprBase"},
- {"Index": 4, "Name": "ListHandlerLambda", "Type": "TCoLambda"}
- ]
- },
- {
- "Name": "TCoPartitionByKey",
- "Base": "TCoPartitionByKeyBase",
- "Match": {"Type": "Callable", "Name": "PartitionByKey"}
- },
- {
+ {"Index": 1, "Name": "KeySelectorLambda", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "HandlerLambda", "Type": "TCoLambda"}
+ ]
+ },
+ {
+ "Name": "TCoPartitionByKeyBase",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "CallableBase"},
+ "Children": [
+ {"Index": 1, "Name": "KeySelectorLambda", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "SortDirections", "Type": "TExprBase"},
+ {"Index": 3, "Name": "SortKeySelectorLambda", "Type": "TExprBase"},
+ {"Index": 4, "Name": "ListHandlerLambda", "Type": "TCoLambda"}
+ ]
+ },
+ {
+ "Name": "TCoPartitionByKey",
+ "Base": "TCoPartitionByKeyBase",
+ "Match": {"Type": "Callable", "Name": "PartitionByKey"}
+ },
+ {
"Name": "TCoPartitionsByKeys",
- "Base": "TCoPartitionByKeyBase",
- "Match": {"Type": "Callable", "Name": "PartitionsByKeys"}
+ "Base": "TCoPartitionByKeyBase",
+ "Match": {"Type": "Callable", "Name": "PartitionsByKeys"}
},
{
"Name": "TCoSortTraits",
@@ -548,7 +548,7 @@
]
},
{
- "Name": "TCoHoppingTraits",
+ "Name": "TCoHoppingTraits",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "HoppingTraits"},
"Children": [
@@ -573,51 +573,51 @@
]
},
{
- "Name": "TCoAggregationTraits",
+ "Name": "TCoAggregationTraits",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "AggregationTraits"},
"Children": [
{"Index": 0, "Name": "ItemType", "Type": "TExprBase"},
- {"Index": 1, "Name": "InitHandler", "Type": "TCoLambda"},
- {"Index": 2, "Name": "UpdateHandler", "Type": "TCoLambda"},
- {"Index": 3, "Name": "SaveHandler", "Type": "TCoLambda"},
- {"Index": 4, "Name": "LoadHandler", "Type": "TCoLambda"},
- {"Index": 5, "Name": "MergeHandler", "Type": "TCoLambda"},
+ {"Index": 1, "Name": "InitHandler", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "UpdateHandler", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "SaveHandler", "Type": "TCoLambda"},
+ {"Index": 4, "Name": "LoadHandler", "Type": "TCoLambda"},
+ {"Index": 5, "Name": "MergeHandler", "Type": "TCoLambda"},
{"Index": 6, "Name": "FinishHandler", "Type": "TCoLambda"},
{"Index": 7, "Name": "DefVal", "Type": "TExprBase"}
]
},
{
- "Name": "TCoAggregateTuple",
+ "Name": "TCoAggregateTuple",
"Base": "TExprBase",
"Match": {"Type": "Tuple"},
"Children": [
{"Index": 0, "Name": "ColumnName", "Type": "TExprBase"},
- {"Index": 1, "Name": "Trait", "Type": "TCoAggregationTraits"},
- {"Index": 2, "Name": "DistinctName", "Type": "TCoAtom", "Optional": true}
+ {"Index": 1, "Name": "Trait", "Type": "TCoAggregationTraits"},
+ {"Index": 2, "Name": "DistinctName", "Type": "TCoAtom", "Optional": true}
]
},
{
- "Name": "TCoAggregateTupleList",
- "ListBase": "TCoAggregateTuple"
+ "Name": "TCoAggregateTupleList",
+ "ListBase": "TCoAggregateTuple"
},
{
- "Name": "TCoAggregate",
- "Base": "TCoInputBase",
+ "Name": "TCoAggregate",
+ "Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "Aggregate"},
"Children": [
- {"Index": 1, "Name": "Keys", "Type": "TCoAtomList"},
- {"Index": 2, "Name": "Handlers", "Type": "TCoAggregateTupleList"},
+ {"Index": 1, "Name": "Keys", "Type": "TCoAtomList"},
+ {"Index": 2, "Name": "Handlers", "Type": "TCoAggregateTupleList"},
{"Index": 3, "Name": "Settings", "Type": "TCoNameValueTupleList"}
]
},
{
- "Name": "TCoFold1",
- "Base": "TCoInputBase",
+ "Name": "TCoFold1",
+ "Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "Fold1"},
"Children": [
- {"Index": 1, "Name": "InitHandler", "Type": "TCoLambda"},
- {"Index": 2, "Name": "UpdateHandler", "Type": "TCoLambda"}
+ {"Index": 1, "Name": "InitHandler", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "UpdateHandler", "Type": "TCoLambda"}
]
},
{
@@ -630,99 +630,99 @@
]
},
{
- "Name": "TCoZip",
- "VarArgBase": "TExprBase",
+ "Name": "TCoZip",
+ "VarArgBase": "TExprBase",
"Match": {"Type": "Callable", "Name": "Zip"}
},
{
- "Name": "TCoZipAll",
- "VarArgBase": "TExprBase",
- "Match": {"Type": "Callable", "Name": "ZipAll"}
- },
- {
- "Name": "TCoSync",
- "VarArgBase": "TExprBase",
+ "Name": "TCoZipAll",
+ "VarArgBase": "TExprBase",
+ "Match": {"Type": "Callable", "Name": "ZipAll"}
+ },
+ {
+ "Name": "TCoSync",
+ "VarArgBase": "TExprBase",
"Match": {"Type": "Callable", "Name": "Sync!"}
},
{
- "Name": "TCoDataSource",
+ "Name": "TCoDataSource",
"Base": "TFreeArgCallable",
"Match": {"Type": "Callable", "Name": "DataSource"},
"Children": [
- {"Index": 0, "Name": "Category", "Type": "TCoAtom"}
+ {"Index": 0, "Name": "Category", "Type": "TCoAtom"}
]
},
{
- "Name": "TCoDataSink",
+ "Name": "TCoDataSink",
"Base": "TFreeArgCallable",
"Match": {"Type": "Callable", "Name": "DataSink"},
"Children": [
- {"Index": 0, "Name": "Category", "Type": "TCoAtom"}
+ {"Index": 0, "Name": "Category", "Type": "TCoAtom"}
]
},
{
- "Name": "TCoCommit",
+ "Name": "TCoCommit",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Commit!"},
"Children": [
{"Index": 0, "Name": "World", "Type": "TExprBase"},
{"Index": 1, "Name": "DataSink", "Type": "TExprBase"},
- {"Index": 2, "Name": "Settings", "Type": "TCoNameValueTupleList", "Optional": true}
+ {"Index": 2, "Name": "Settings", "Type": "TCoNameValueTupleList", "Optional": true}
]
},
{
- "Name": "TCoWrite",
+ "Name": "TCoWrite",
"Base": "TFreeArgCallable",
"Match": {"Type": "Callable", "Name": "Write!"},
"Children": [
{"Index": 0, "Name": "World", "Type": "TExprBase"},
- {"Index": 1, "Name": "DataSink", "Type": "TCoDataSink"}
+ {"Index": 1, "Name": "DataSink", "Type": "TCoDataSink"}
]
},
{
- "Name": "TCoKey",
- "VarArgBase": "TExprBase",
+ "Name": "TCoKey",
+ "VarArgBase": "TExprBase",
"Match": {"Type": "Callable", "Name": "Key"}
},
{
- "Name": "TCoRead",
+ "Name": "TCoRead",
"Base": "TFreeArgCallable",
"Match": {"Type": "Callable", "Name": "Read!"},
"Children": [
{"Index": 0, "Name": "World", "Type": "TExprBase"},
- {"Index": 1, "Name": "DataSource", "Type": "TCoDataSource"}
- ]
- },
- {
- "Name": "TCoConfigure",
- "Base": "TFreeArgCallable",
- "Match": {"Type": "Callable", "Name": "Configure!"},
- "Children": [
- {"Index": 0, "Name": "World", "Type": "TExprBase"},
- {"Index": 1, "Name": "DataSource", "Type": "TCoDataSource"}
- ]
- },
- {
- "Name": "TCoDataType",
+ {"Index": 1, "Name": "DataSource", "Type": "TCoDataSource"}
+ ]
+ },
+ {
+ "Name": "TCoConfigure",
+ "Base": "TFreeArgCallable",
+ "Match": {"Type": "Callable", "Name": "Configure!"},
+ "Children": [
+ {"Index": 0, "Name": "World", "Type": "TExprBase"},
+ {"Index": 1, "Name": "DataSource", "Type": "TCoDataSource"}
+ ]
+ },
+ {
+ "Name": "TCoDataType",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "DataType"},
"Children": [
- {"Index": 0, "Name": "Type", "Type": "TCoAtom"},
- {"Index": 1, "Name": "ParamOne", "Type": "TCoAtom", "Optional": true},
- {"Index": 2, "Name": "ParamTwo", "Type": "TCoAtom", "Optional": true}
+ {"Index": 0, "Name": "Type", "Type": "TCoAtom"},
+ {"Index": 1, "Name": "ParamOne", "Type": "TCoAtom", "Optional": true},
+ {"Index": 2, "Name": "ParamTwo", "Type": "TCoAtom", "Optional": true}
]
},
{
- "Name": "TCoTaggedType",
+ "Name": "TCoTaggedType",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "TaggedType"},
"Children": [
{"Index": 0, "Name": "Type", "Type": "TExprBase"},
- {"Index": 1, "Name": "Tag", "Type": "TCoAtom"}
+ {"Index": 1, "Name": "Tag", "Type": "TCoAtom"}
]
},
{
- "Name": "TCoListType",
+ "Name": "TCoListType",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "ListType"},
"Children": [
@@ -730,12 +730,12 @@
]
},
{
- "Name": "TCoStructType",
- "VarArgBase": "TExprBase",
- "Match": {"Type": "Callable", "Name": "StructType"}
- },
- {
- "Name": "TCoOptionalType",
+ "Name": "TCoStructType",
+ "VarArgBase": "TExprBase",
+ "Match": {"Type": "Callable", "Name": "StructType"}
+ },
+ {
+ "Name": "TCoOptionalType",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "OptionalType"},
"Children": [
@@ -743,181 +743,181 @@
]
},
{
- "Name": "TCoVoidType",
+ "Name": "TCoVoidType",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "VoidType"}
},
{
- "Name": "TCoTupleType",
- "VarArgBase": "TExprBase",
- "Match": {"Type": "Callable", "Name": "TupleType"}
- },
- {
- "Name": "TCoVariantType",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "VariantType"},
- "Children": [
- {"Index": 0, "Name": "UnderlyingType", "Type": "TExprBase"}
- ]
- },
- {
- "Name": "TCoDataCtor",
+ "Name": "TCoTupleType",
+ "VarArgBase": "TExprBase",
+ "Match": {"Type": "Callable", "Name": "TupleType"}
+ },
+ {
+ "Name": "TCoVariantType",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "VariantType"},
+ "Children": [
+ {"Index": 0, "Name": "UnderlyingType", "Type": "TExprBase"}
+ ]
+ },
+ {
+ "Name": "TCoDataCtor",
"Base": "TCallable",
"Match": {"Type": "CallableBase"},
"Builder": {"Generate": "None"},
"Children": [
- {"Index": 0, "Name": "Literal", "Type": "TCoAtom"}
+ {"Index": 0, "Name": "Literal", "Type": "TCoAtom"}
]
},
{
- "Name": "TCoIntegralCtor",
- "Base": "TCoDataCtor",
+ "Name": "TCoIntegralCtor",
+ "Base": "TCoDataCtor",
"Match": {"Type": "CallableBase"},
"Builder": {"Generate": "None"}
},
{
- "Name": "TCoUint8",
- "Base": "TCoIntegralCtor",
+ "Name": "TCoUint8",
+ "Base": "TCoIntegralCtor",
"Match": {"Type": "Callable", "Name": "Uint8"}
},
{
- "Name": "TCoInt8",
- "Base": "TCoIntegralCtor",
+ "Name": "TCoInt8",
+ "Base": "TCoIntegralCtor",
"Match": {"Type": "Callable", "Name": "Int8"}
},
{
- "Name": "TCoUint16",
- "Base": "TCoIntegralCtor",
+ "Name": "TCoUint16",
+ "Base": "TCoIntegralCtor",
"Match": {"Type": "Callable", "Name": "Uint16"}
},
{
- "Name": "TCoInt16",
- "Base": "TCoIntegralCtor",
+ "Name": "TCoInt16",
+ "Base": "TCoIntegralCtor",
"Match": {"Type": "Callable", "Name": "Int16"}
},
{
- "Name": "TCoInt32",
- "Base": "TCoIntegralCtor",
+ "Name": "TCoInt32",
+ "Base": "TCoIntegralCtor",
"Match": {"Type": "Callable", "Name": "Int32"}
},
{
- "Name": "TCoUint32",
- "Base": "TCoIntegralCtor",
+ "Name": "TCoUint32",
+ "Base": "TCoIntegralCtor",
"Match": {"Type": "Callable", "Name": "Uint32"}
},
{
- "Name": "TCoInt64",
- "Base": "TCoIntegralCtor",
+ "Name": "TCoInt64",
+ "Base": "TCoIntegralCtor",
"Match": {"Type": "Callable", "Name": "Int64"}
},
{
- "Name": "TCoUint64",
- "Base": "TCoIntegralCtor",
+ "Name": "TCoUint64",
+ "Base": "TCoIntegralCtor",
"Match": {"Type": "Callable", "Name": "Uint64"}
},
{
- "Name": "TCoFloat",
- "Base": "TCoDataCtor",
- "Match": {"Type": "Callable", "Name": "Float"}
- },
- {
- "Name": "TCoDouble",
- "Base": "TCoDataCtor",
- "Match": {"Type": "Callable", "Name": "Double"}
- },
- {
- "Name": "TCoString",
- "Base": "TCoDataCtor",
+ "Name": "TCoFloat",
+ "Base": "TCoDataCtor",
+ "Match": {"Type": "Callable", "Name": "Float"}
+ },
+ {
+ "Name": "TCoDouble",
+ "Base": "TCoDataCtor",
+ "Match": {"Type": "Callable", "Name": "Double"}
+ },
+ {
+ "Name": "TCoString",
+ "Base": "TCoDataCtor",
"Match": {"Type": "Callable", "Name": "String"}
},
{
- "Name": "TCoUtf8",
- "Base": "TCoDataCtor",
+ "Name": "TCoUtf8",
+ "Base": "TCoDataCtor",
"Match": {"Type": "Callable", "Name": "Utf8"}
},
{
- "Name": "TCoYson",
- "Base": "TCoDataCtor",
- "Match": {"Type": "Callable", "Name": "Yson"}
- },
- {
- "Name": "TCoJson",
- "Base": "TCoDataCtor",
- "Match": {"Type": "Callable", "Name": "Json"}
- },
- {
- "Name": "TCoBool",
- "Base": "TCoDataCtor",
+ "Name": "TCoYson",
+ "Base": "TCoDataCtor",
+ "Match": {"Type": "Callable", "Name": "Yson"}
+ },
+ {
+ "Name": "TCoJson",
+ "Base": "TCoDataCtor",
+ "Match": {"Type": "Callable", "Name": "Json"}
+ },
+ {
+ "Name": "TCoBool",
+ "Base": "TCoDataCtor",
"Match": {"Type": "Callable", "Name": "Bool"}
},
{
- "Name": "TCoDateBase",
- "Base": "TCoDataCtor",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"}
- },
- {
- "Name": "TCoDate",
- "Base": "TCoDateBase",
- "Match": {"Type": "Callable", "Name": "Date"}
- },
- {
- "Name": "TCoDatetime",
- "Base": "TCoDateBase",
- "Match": {"Type": "Callable", "Name": "Datetime"}
- },
- {
- "Name": "TCoTimestamp",
- "Base": "TCoDateBase",
- "Match": {"Type": "Callable", "Name": "Timestamp"}
- },
- {
- "Name": "TCoTzDateBase",
- "Base": "TCoDateBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"}
- },
- {
- "Name": "TCoTzDate",
- "Base": "TCoTzDateBase",
- "Match": {"Type": "Callable", "Name": "TzDate"}
- },
- {
- "Name": "TCoTzDatetime",
- "Base": "TCoTzDateBase",
- "Match": {"Type": "Callable", "Name": "TzDatetime"}
- },
- {
- "Name": "TCoTzTimestamp",
- "Base": "TCoTzDateBase",
- "Match": {"Type": "Callable", "Name": "TzTimestamp"}
- },
- {
- "Name": "TCoInterval",
- "Base": "TCoDataCtor",
- "Match": {"Type": "Callable", "Name": "Interval"}
- },
- {
- "Name": "TCoDecimal",
- "Base": "TCoDataCtor",
- "Match": {"Type": "Callable", "Name": "Decimal"},
- "Children": [
- {"Index": 1, "Name": "Precision", "Type": "TCoAtom"},
- {"Index": 2, "Name": "Scale", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TCoUuid",
- "Base": "TCoDataCtor",
- "Match": {"Type": "Callable", "Name": "Uuid"}
- },
- {
- "Name": "TCoAsList",
- "VarArgBase": "TExprBase",
+ "Name": "TCoDateBase",
+ "Base": "TCoDataCtor",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"}
+ },
+ {
+ "Name": "TCoDate",
+ "Base": "TCoDateBase",
+ "Match": {"Type": "Callable", "Name": "Date"}
+ },
+ {
+ "Name": "TCoDatetime",
+ "Base": "TCoDateBase",
+ "Match": {"Type": "Callable", "Name": "Datetime"}
+ },
+ {
+ "Name": "TCoTimestamp",
+ "Base": "TCoDateBase",
+ "Match": {"Type": "Callable", "Name": "Timestamp"}
+ },
+ {
+ "Name": "TCoTzDateBase",
+ "Base": "TCoDateBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"}
+ },
+ {
+ "Name": "TCoTzDate",
+ "Base": "TCoTzDateBase",
+ "Match": {"Type": "Callable", "Name": "TzDate"}
+ },
+ {
+ "Name": "TCoTzDatetime",
+ "Base": "TCoTzDateBase",
+ "Match": {"Type": "Callable", "Name": "TzDatetime"}
+ },
+ {
+ "Name": "TCoTzTimestamp",
+ "Base": "TCoTzDateBase",
+ "Match": {"Type": "Callable", "Name": "TzTimestamp"}
+ },
+ {
+ "Name": "TCoInterval",
+ "Base": "TCoDataCtor",
+ "Match": {"Type": "Callable", "Name": "Interval"}
+ },
+ {
+ "Name": "TCoDecimal",
+ "Base": "TCoDataCtor",
+ "Match": {"Type": "Callable", "Name": "Decimal"},
+ "Children": [
+ {"Index": 1, "Name": "Precision", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "Scale", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TCoUuid",
+ "Base": "TCoDataCtor",
+ "Match": {"Type": "Callable", "Name": "Uuid"}
+ },
+ {
+ "Name": "TCoAsList",
+ "VarArgBase": "TExprBase",
"Match": {"Type": "Callable", "Name": "AsList"}
},
{
- "Name": "TCoToList",
+ "Name": "TCoToList",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "ToList"},
"Children": [
@@ -925,7 +925,7 @@
]
},
{
- "Name": "TCoToOptional",
+ "Name": "TCoToOptional",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "ToOptional"},
"Children": [
@@ -933,16 +933,16 @@
]
},
{
- "Name": "TCoList",
- "Base": "TFreeArgCallable",
+ "Name": "TCoList",
+ "Base": "TFreeArgCallable",
"Match": {"Type": "Callable", "Name": "List"},
"Children": [
{"Index": 0, "Name": "ListType", "Type": "TExprBase"}
]
},
{
- "Name": "TCoAsStruct",
- "VarArgBase": "TExprList",
+ "Name": "TCoAsStruct",
+ "VarArgBase": "TExprList",
"Match": {"Type": "Callable", "Name": "AsStruct"}
},
{
@@ -951,18 +951,18 @@
"Match": {"Type": "Callable", "Name": "AsDict"}
},
{
- "Name": "TCoToDict",
+ "Name": "TCoToDict",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "ToDict"},
"Children": [
{"Index": 0, "Name": "List", "Type": "TExprBase"},
- {"Index": 1, "Name": "KeySelector", "Type": "TCoLambda"},
- {"Index": 2, "Name": "PayloadSelector", "Type": "TCoLambda"},
- {"Index": 3, "Name": "Settings", "Type": "TCoAtomList"}
+ {"Index": 1, "Name": "KeySelector", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "PayloadSelector", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "Settings", "Type": "TCoAtomList"}
]
},
{
- "Name": "TCoDictFromKeys",
+ "Name": "TCoDictFromKeys",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "DictFromKeys"},
"Children": [
@@ -1027,7 +1027,7 @@
{
"Name": "TCoContains",
"Base": "TCoLookupBase",
- "Match": {"Type": "Callable", "Name": "Contains"}
+ "Match": {"Type": "Callable", "Name": "Contains"}
},
{
"Name": "TCoSqlIn",
@@ -1038,30 +1038,30 @@
]
},
{
- "Name": "TCoLookup",
+ "Name": "TCoLookup",
"Base": "TCoLookupBase",
"Match": {"Type": "Callable", "Name": "Lookup"}
},
{
- "Name": "TCoMember",
+ "Name": "TCoMember",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Member"},
"Children": [
{"Index": 0, "Name": "Struct", "Type": "TExprBase"},
- {"Index": 1, "Name": "Name", "Type": "TCoAtom"}
+ {"Index": 1, "Name": "Name", "Type": "TCoAtom"}
]
},
{
- "Name": "TCoNth",
+ "Name": "TCoNth",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Nth"},
"Children": [
{"Index": 0, "Name": "Tuple", "Type": "TExprBase"},
- {"Index": 1, "Name": "Index", "Type": "TCoAtom"}
+ {"Index": 1, "Name": "Index", "Type": "TCoAtom"}
]
},
{
- "Name": "TCoCompare",
+ "Name": "TCoCompare",
"Base": "TCallable",
"Match": {"Type": "CallableBase"},
"Builder": {"Generate": "None"},
@@ -1071,33 +1071,33 @@
]
},
{
- "Name": "TCoCmpLess",
- "Base": "TCoCompare",
+ "Name": "TCoCmpLess",
+ "Base": "TCoCompare",
"Match": {"Type": "Callable", "Name": "<"}
},
{
- "Name": "TCoCmpLessOrEqual",
- "Base": "TCoCompare",
+ "Name": "TCoCmpLessOrEqual",
+ "Base": "TCoCompare",
"Match": {"Type": "Callable", "Name": "<="}
},
{
- "Name": "TCoCmpGreater",
- "Base": "TCoCompare",
+ "Name": "TCoCmpGreater",
+ "Base": "TCoCompare",
"Match": {"Type": "Callable", "Name": ">"}
},
{
- "Name": "TCoCmpGreaterOrEqual",
- "Base": "TCoCompare",
+ "Name": "TCoCmpGreaterOrEqual",
+ "Base": "TCoCompare",
"Match": {"Type": "Callable", "Name": ">="}
},
{
- "Name": "TCoCmpEqual",
- "Base": "TCoCompare",
+ "Name": "TCoCmpEqual",
+ "Base": "TCoCompare",
"Match": {"Type": "Callable", "Name": "=="}
},
{
- "Name": "TCoCmpNotEqual",
- "Base": "TCoCompare",
+ "Name": "TCoCmpNotEqual",
+ "Base": "TCoCompare",
"Match": {"Type": "Callable", "Name": "!="}
},
{
@@ -1137,7 +1137,7 @@
]
},
{
- "Name": "TCoNot",
+ "Name": "TCoNot",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Not"},
"Children": [
@@ -1145,12 +1145,12 @@
]
},
{
- "Name": "TCoAnd",
+ "Name": "TCoAnd",
"VarArgBase": "TExprBase",
"Match": {"Type": "Callable", "Name": "And"}
},
{
- "Name": "TCoOr",
+ "Name": "TCoOr",
"VarArgBase": "TExprBase",
"Match": {"Type": "Callable", "Name": "Or"}
},
@@ -1160,31 +1160,31 @@
"Match": {"Type": "Callable", "Name": "Xor"}
},
{
- "Name": "TCoBinaryArithmetic",
- "Base": "TCallable",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
- "Children": [
- {"Index": 0, "Name": "Left", "Type": "TExprBase"},
- {"Index": 1, "Name": "Right", "Type": "TExprBase"}
- ]
- },
- {
+ "Name": "TCoBinaryArithmetic",
+ "Base": "TCallable",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
+ "Children": [
+ {"Index": 0, "Name": "Left", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Right", "Type": "TExprBase"}
+ ]
+ },
+ {
"Name": "TCoAggrAdd",
"Base": "TCoBinaryArithmetic",
"Match": {"Type": "Callable", "Name": "AggrAdd"}
},
{
- "Name": "TCoMinus",
- "Base": "TCoBinaryArithmetic",
- "Match": {"Type": "Callable", "Name": "-"}
- },
- {
- "Name": "TCoPlus",
- "Base": "TCoBinaryArithmetic",
- "Match": {"Type": "Callable", "Name": "+"}
- },
- {
+ "Name": "TCoMinus",
+ "Base": "TCoBinaryArithmetic",
+ "Match": {"Type": "Callable", "Name": "-"}
+ },
+ {
+ "Name": "TCoPlus",
+ "Base": "TCoBinaryArithmetic",
+ "Match": {"Type": "Callable", "Name": "+"}
+ },
+ {
"Name": "TCoMul",
"Base": "TCoBinaryArithmetic",
"Match": {"Type": "Callable", "Name": "*"}
@@ -1200,7 +1200,7 @@
"Match": {"Type": "Callable", "Name": "%"}
},
{
- "Name": "TCoCoalesce",
+ "Name": "TCoCoalesce",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Coalesce"},
"Children": [
@@ -1209,7 +1209,7 @@
]
},
{
- "Name": "TCoUnwrap",
+ "Name": "TCoUnwrap",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Unwrap"},
"Children": [
@@ -1217,15 +1217,15 @@
]
},
{
- "Name": "TCoScriptUdf",
+ "Name": "TCoScriptUdf",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "ScriptUdf"},
"Children": [
- {"Index": 0, "Name": "ScriptType", "Type": "TCoAtom"},
- {"Index": 1, "Name": "FunctionName", "Type": "TCoAtom"},
+ {"Index": 0, "Name": "ScriptType", "Type": "TCoAtom"},
+ {"Index": 1, "Name": "FunctionName", "Type": "TCoAtom"},
{"Index": 2, "Name": "FunctionType", "Type": "TExprBase"},
{"Index": 3, "Name": "Script", "Type": "TExprBase"},
- {"Index": 4, "Name": "Options", "Type": "TCoNameValueTupleList", "Optional": true}
+ {"Index": 4, "Name": "Options", "Type": "TCoNameValueTupleList", "Optional": true}
]
},
{
@@ -1243,7 +1243,7 @@
]
},
{
- "Name": "TCoExists",
+ "Name": "TCoExists",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Exists"},
"Children": [
@@ -1251,7 +1251,7 @@
]
},
{
- "Name": "TCoIf",
+ "Name": "TCoIf",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "If"},
"Children": [
@@ -1271,17 +1271,17 @@
]
},
{
- "Name": "TCoIfPresent",
+ "Name": "TCoIfPresent",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "IfPresent"},
"Children": [
{"Index": 0, "Name": "Optional", "Type": "TExprBase"},
- {"Index": 1, "Name": "PresentHandler", "Type": "TCoLambda"},
+ {"Index": 1, "Name": "PresentHandler", "Type": "TCoLambda"},
{"Index": 2, "Name": "MissingValue", "Type": "TExprBase"}
]
},
{
- "Name": "TCoDefault",
+ "Name": "TCoDefault",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Default"},
"Children": [
@@ -1290,11 +1290,11 @@
},
{
"Name": "TCoCalcOverWindowBase",
- "Base": "TCoInputBase",
+ "Base": "TCoInputBase",
"Match": {"Type": "CallableBase"},
"Builder": {"Generate": "None"},
"Children": [
- {"Index": 1, "Name": "Keys", "Type": "TCoAtomList"},
+ {"Index": 1, "Name": "Keys", "Type": "TCoAtomList"},
{"Index": 2, "Name": "SortSpec", "Type": "TExprBase"},
{"Index": 3, "Name": "Frames", "Type": "TExprList"}
]
@@ -1338,20 +1338,20 @@
]
},
{
- "Name": "TCoEquiJoinTuple",
+ "Name": "TCoEquiJoinTuple",
"Base": "TExprBase",
"Match": {"Type": "Tuple"},
"Children": [
- {"Index": 0, "Name": "Type", "Type": "TCoAtom"},
+ {"Index": 0, "Name": "Type", "Type": "TCoAtom"},
{"Index": 1, "Name": "LeftScope", "Type": "TExprBase"},
{"Index": 2, "Name": "RightScope", "Type": "TExprBase"},
- {"Index": 3, "Name": "LeftKeys", "Type": "TCoAtomList"},
- {"Index": 4, "Name": "RightKeys", "Type": "TCoAtomList"},
+ {"Index": 3, "Name": "LeftKeys", "Type": "TCoAtomList"},
+ {"Index": 4, "Name": "RightKeys", "Type": "TCoAtomList"},
{"Index": 5, "Name": "Options", "Type": "TExprList"}
]
},
{
- "Name": "TCoEquiJoinInput",
+ "Name": "TCoEquiJoinInput",
"Base": "TExprBase",
"Match": {"Type": "Tuple"},
"Children": [
@@ -1360,11 +1360,11 @@
]
},
{
- "Name": "TCoEquiJoin",
- "VarArgBase": "TExprBase",
+ "Name": "TCoEquiJoin",
+ "VarArgBase": "TExprBase",
"Match": {"Type": "Callable", "Name": "EquiJoin"}
- },
- {
+ },
+ {
"Name": "TCoMapJoinCore",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "MapJoinCore"},
@@ -1389,47 +1389,47 @@
]
},
{
- "Name": "TCoSortBase",
- "Base": "TCoInputBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
- "Children": [
- {"Index": 1, "Name": "SortDirections", "Type": "TExprBase"},
- {"Index": 2, "Name": "KeySelectorLambda", "Type": "TCoLambda"}
- ]
- },
- {
- "Name": "TCoSort",
- "Base": "TCoSortBase",
- "Match": {"Type": "Callable", "Name": "Sort"}
- },
- {
- "Name": "TCoAssumeSorted",
- "Base": "TCoSortBase",
- "Match": {"Type": "Callable", "Name": "AssumeSorted"}
- },
- {
- "Name": "TCoTopBase",
- "Base": "TCoInputBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
- "Children": [
- {"Index": 1, "Name": "Count", "Type": "TExprBase"},
- {"Index": 2, "Name": "SortDirections", "Type": "TExprBase"},
- {"Index": 3, "Name": "KeySelectorLambda", "Type": "TCoLambda"}
- ]
- },
- {
- "Name": "TCoTop",
- "Base": "TCoTopBase",
- "Match": {"Type": "Callable", "Name": "Top"}
- },
- {
- "Name": "TCoTopSort",
- "Base": "TCoTopBase",
- "Match": {"Type": "Callable", "Name": "TopSort"}
- },
- {
+ "Name": "TCoSortBase",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
+ "Children": [
+ {"Index": 1, "Name": "SortDirections", "Type": "TExprBase"},
+ {"Index": 2, "Name": "KeySelectorLambda", "Type": "TCoLambda"}
+ ]
+ },
+ {
+ "Name": "TCoSort",
+ "Base": "TCoSortBase",
+ "Match": {"Type": "Callable", "Name": "Sort"}
+ },
+ {
+ "Name": "TCoAssumeSorted",
+ "Base": "TCoSortBase",
+ "Match": {"Type": "Callable", "Name": "AssumeSorted"}
+ },
+ {
+ "Name": "TCoTopBase",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
+ "Children": [
+ {"Index": 1, "Name": "Count", "Type": "TExprBase"},
+ {"Index": 2, "Name": "SortDirections", "Type": "TExprBase"},
+ {"Index": 3, "Name": "KeySelectorLambda", "Type": "TCoLambda"}
+ ]
+ },
+ {
+ "Name": "TCoTop",
+ "Base": "TCoTopBase",
+ "Match": {"Type": "Callable", "Name": "Top"}
+ },
+ {
+ "Name": "TCoTopSort",
+ "Base": "TCoTopBase",
+ "Match": {"Type": "Callable", "Name": "TopSort"}
+ },
+ {
"Name": "TCoKeepTop",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "KeepTop"},
@@ -1446,71 +1446,71 @@
"Match": {"Type": "Callable", "Name": "Ascending"}
},
{
- "Name": "TCoDescending",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "Descending"}
- },
- {
- "Name": "TCoAddMember",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "AddMember"},
- "Children": [
- {"Index": 0, "Name": "Struct", "Type": "TExprBase"},
- {"Index": 1, "Name": "Name", "Type": "TCoAtom"},
- {"Index": 2, "Name": "Item", "Type": "TExprBase"}
- ]
+ "Name": "TCoDescending",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "Descending"}
+ },
+ {
+ "Name": "TCoAddMember",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "AddMember"},
+ "Children": [
+ {"Index": 0, "Name": "Struct", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Name", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "Item", "Type": "TExprBase"}
+ ]
},
{
- "Name": "TCoReplaceMember",
+ "Name": "TCoReplaceMember",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "ReplaceMember"},
"Children": [
{"Index": 0, "Name": "Struct", "Type": "TExprBase"},
- {"Index": 1, "Name": "Name", "Type": "TCoAtom"},
+ {"Index": 1, "Name": "Name", "Type": "TCoAtom"},
{"Index": 2, "Name": "Item", "Type": "TExprBase"}
]
},
{
- "Name": "TCoRemoveMember",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "RemoveMember"},
- "Children": [
- {"Index": 0, "Name": "Struct", "Type": "TExprBase"},
- {"Index": 1, "Name": "Name", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TCoForceRemoveMember",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "ForceRemoveMember"},
- "Children": [
- {"Index": 0, "Name": "Struct", "Type": "TExprBase"},
- {"Index": 1, "Name": "Name", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TCoExtractMembers",
- "Base": "TCoInputBase",
+ "Name": "TCoRemoveMember",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "RemoveMember"},
+ "Children": [
+ {"Index": 0, "Name": "Struct", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Name", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TCoForceRemoveMember",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "ForceRemoveMember"},
+ "Children": [
+ {"Index": 0, "Name": "Struct", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Name", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TCoExtractMembers",
+ "Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "ExtractMembers"},
"Children": [
- {"Index": 1, "Name": "Members", "Type": "TCoAtomList"}
+ {"Index": 1, "Name": "Members", "Type": "TCoAtomList"}
]
- },
+ },
- {
+ {
"Name": "TCoFlattenMembers",
"VarArgBase": "TCoNameValueTuple",
"Match": {"Type": "Callable", "Name": "FlattenMembers"}
},
{
- "Name": "TCoAssumeUnique",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "AssumeUnique"},
- "Children": [
- {"Index": 1, "Name": "UniqueBy", "Type": "TCoAtomList"}
- ]
- },
- {
+ "Name": "TCoAssumeUnique",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "AssumeUnique"},
+ "Children": [
+ {"Index": 1, "Name": "UniqueBy", "Type": "TCoAtomList"}
+ ]
+ },
+ {
"Name": "TCoAssumeColumnOrder",
"Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "AssumeColumnOrder"},
@@ -1527,17 +1527,17 @@
]
},
{
- "Name": "TCoAssumeAllMembersNullableAtOnce",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "AssumeAllMembersNullableAtOnce"}
- },
- {
- "Name": "TCoListItemType",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "ListItemType"},
- "Children": [
- {"Index": 0, "Name": "ListType", "Type": "TExprBase"}
- ]
+ "Name": "TCoAssumeAllMembersNullableAtOnce",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "AssumeAllMembersNullableAtOnce"}
+ },
+ {
+ "Name": "TCoListItemType",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "ListItemType"},
+ "Children": [
+ {"Index": 0, "Name": "ListType", "Type": "TExprBase"}
+ ]
},
{
"Name": "TCoStreamItemType",
@@ -1556,32 +1556,32 @@
]
},
{
- "Name": "TCoDependsOn",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "DependsOn"}
- },
- {
- "Name": "TCoParameter",
+ "Name": "TCoDependsOn",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "DependsOn"}
+ },
+ {
+ "Name": "TCoParameter",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Parameter"},
"Children": [
- {"Index": 0, "Name": "Name", "Type": "TCoAtom"},
+ {"Index": 0, "Name": "Name", "Type": "TCoAtom"},
{"Index": 1, "Name": "Type", "Type": "TExprBase"}
]
},
{
- "Name": "TCoMatchType",
+ "Name": "TCoMatchType",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "MatchType"},
"Children": [
{"Index": 0, "Name": "Expression", "Type": "TExprBase"},
- {"Index": 1, "Name": "TypeKind", "Type": "TCoAtom"},
- {"Index": 2, "Name": "MatchLambda", "Type": "TCoLambda"},
- {"Index": 3, "Name": "NotMatchLambda", "Type": "TCoLambda"}
+ {"Index": 1, "Name": "TypeKind", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "MatchLambda", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "NotMatchLambda", "Type": "TCoLambda"}
]
},
{
- "Name": "TCoIterator",
+ "Name": "TCoIterator",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "Iterator"},
"Children": [
@@ -1589,7 +1589,7 @@
]
},
{
- "Name": "TCoForwardList",
+ "Name": "TCoForwardList",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "ForwardList"},
"Children": [
@@ -1597,8 +1597,8 @@
]
},
{
- "Name": "TCoHoppingCore",
- "Base": "TCoInputBase",
+ "Name": "TCoHoppingCore",
+ "Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "HoppingCore"},
"Children": [
{"Index": 1, "Name": "TimeExtractor", "Type": "TCoLambda"},
@@ -1633,18 +1633,18 @@
]
},
{
- "Name": "TCoCombineCore",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "CombineCore"},
- "Children": [
- {"Index": 1, "Name": "KeyExtractor", "Type": "TCoLambda"},
- {"Index": 2, "Name": "InitHandler", "Type": "TCoLambda"},
- {"Index": 3, "Name": "UpdateHandler", "Type": "TCoLambda"},
- {"Index": 4, "Name": "FinishHandler", "Type": "TCoLambda"},
+ "Name": "TCoCombineCore",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "CombineCore"},
+ "Children": [
+ {"Index": 1, "Name": "KeyExtractor", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "InitHandler", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "UpdateHandler", "Type": "TCoLambda"},
+ {"Index": 4, "Name": "FinishHandler", "Type": "TCoLambda"},
{"Index": 5, "Name": "MemLimit", "Type": "TCoAtom", "Optional": true}
- ]
- },
- {
+ ]
+ },
+ {
"Name": "TCoWideCombiner",
"Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "WideCombiner"},
@@ -1657,21 +1657,21 @@
]
},
{
- "Name": "TCoGroupingCore",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "GroupingCore"},
- "Children": [
- {"Index": 1, "Name": "GroupSwitch", "Type": "TCoLambda"},
- {"Index": 2, "Name": "KeyExtractor", "Type": "TCoLambda"},
- {"Index": 3, "Name": "ConvertHandler", "Type": "TCoLambda", "Optional": true}
- ]
- },
- {
- "Name": "TCoConvert",
- "Base": "TCoInputBase",
+ "Name": "TCoGroupingCore",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "GroupingCore"},
+ "Children": [
+ {"Index": 1, "Name": "GroupSwitch", "Type": "TCoLambda"},
+ {"Index": 2, "Name": "KeyExtractor", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "ConvertHandler", "Type": "TCoLambda", "Optional": true}
+ ]
+ },
+ {
+ "Name": "TCoConvert",
+ "Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "Convert"},
"Children": [
- {"Index": 1, "Name": "Type", "Type": "TCoAtom"}
+ {"Index": 1, "Name": "Type", "Type": "TCoAtom"}
]
},
{
@@ -1695,91 +1695,91 @@
]
},
{
- "Name": "TCoTablePropBase",
- "Base": "TCallable",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"}
- },
- {
- "Name": "TCoTablePath",
- "Base": "TCoTablePropBase",
- "Match": {"Type": "Callable", "Name": "TablePath"}
- },
- {
+ "Name": "TCoTablePropBase",
+ "Base": "TCallable",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"}
+ },
+ {
+ "Name": "TCoTablePath",
+ "Base": "TCoTablePropBase",
+ "Match": {"Type": "Callable", "Name": "TablePath"}
+ },
+ {
"Name": "TCoTableName",
"Base": "TCoTablePropBase",
"Match": {"Type": "Callable", "Name": "TableName"}
},
{
- "Name": "TCoTableRecord",
- "Base": "TCoTablePropBase",
- "Match": {"Type": "Callable", "Name": "TableRecord"}
- },
- {
- "Name": "TCoIsKeySwitch",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "IsKeySwitch"},
- "Children": [
- {"Index": 0, "Name": "Item", "Type": "TExprBase"},
- {"Index": 1, "Name": "State", "Type": "TExprBase"},
- {"Index": 2, "Name": "ItemKeyExtractor", "Type": "TCoLambda"},
- {"Index": 3, "Name": "StateKeyExtractor", "Type": "TCoLambda"}
- ]
- },
- {
- "Name": "TCoJoin",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "Join"},
- "Children": [
- {"Index": 0, "Name": "LeftInput", "Type": "TExprBase"},
- {"Index": 1, "Name": "RightInput", "Type": "TExprBase"},
- {"Index": 2, "Name": "LeftLambda", "Type": "TCoLambda"},
- {"Index": 3, "Name": "RightLambda", "Type": "TCoLambda"},
- {"Index": 4, "Name": "JoinKind", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TCoEnumerate",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "Enumerate"},
- "Children": [
- {"Index": 1, "Name": "Start", "Type": "TExprBase", "Optional": true},
- {"Index": 2, "Name": "Step", "Type": "TExprBase", "Optional": true}
- ]
- },
- {
+ "Name": "TCoTableRecord",
+ "Base": "TCoTablePropBase",
+ "Match": {"Type": "Callable", "Name": "TableRecord"}
+ },
+ {
+ "Name": "TCoIsKeySwitch",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "IsKeySwitch"},
+ "Children": [
+ {"Index": 0, "Name": "Item", "Type": "TExprBase"},
+ {"Index": 1, "Name": "State", "Type": "TExprBase"},
+ {"Index": 2, "Name": "ItemKeyExtractor", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "StateKeyExtractor", "Type": "TCoLambda"}
+ ]
+ },
+ {
+ "Name": "TCoJoin",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "Join"},
+ "Children": [
+ {"Index": 0, "Name": "LeftInput", "Type": "TExprBase"},
+ {"Index": 1, "Name": "RightInput", "Type": "TExprBase"},
+ {"Index": 2, "Name": "LeftLambda", "Type": "TCoLambda"},
+ {"Index": 3, "Name": "RightLambda", "Type": "TCoLambda"},
+ {"Index": 4, "Name": "JoinKind", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TCoEnumerate",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "Enumerate"},
+ "Children": [
+ {"Index": 1, "Name": "Start", "Type": "TExprBase", "Optional": true},
+ {"Index": 2, "Name": "Step", "Type": "TExprBase", "Optional": true}
+ ]
+ },
+ {
"Name": "TCoDiscard",
"Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "Discard"}
},
{
- "Name": "TCoToStream",
- "Base": "TFreeArgCallable",
- "Match": {"Type": "Callable", "Name": "ToStream"},
- "Children": [
- {"Index": 0, "Name": "Input", "Type": "TExprBase"}
- ]
- },
- {
- "Name": "TCoToFlow",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "ToFlow"}
- },
- {
- "Name": "TCoFromFlow",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "FromFlow"}
- },
- {
+ "Name": "TCoToStream",
+ "Base": "TFreeArgCallable",
+ "Match": {"Type": "Callable", "Name": "ToStream"},
+ "Children": [
+ {"Index": 0, "Name": "Input", "Type": "TExprBase"}
+ ]
+ },
+ {
+ "Name": "TCoToFlow",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "ToFlow"}
+ },
+ {
+ "Name": "TCoFromFlow",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "FromFlow"}
+ },
+ {
"Name": "TCoBitCast",
- "Base": "TCallable",
+ "Base": "TCallable",
"Match": {"Type": "Callable", "Name": "BitCast"},
- "Children": [
- {"Index": 0, "Name": "Value", "Type": "TExprBase"},
- {"Index": 1, "Name": "Type", "Type": "TCoAtom"}
- ]
- },
- {
+ "Children": [
+ {"Index": 0, "Name": "Value", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Type", "Type": "TCoAtom"}
+ ]
+ },
+ {
"Name": "TCoSafeCast",
"Base": "TCallable",
"Match": {"Type": "Callable", "Name": "SafeCast"},
@@ -1798,72 +1798,72 @@
]
},
{
- "Name": "TCoFromYsonSimpleType",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "FromYsonSimpleType"},
- "Children": [
- {"Index": 0, "Name": "Value", "Type": "TExprBase"},
- {"Index": 1, "Name": "Type", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TCoVisit",
- "Base": "TFreeArgCallable",
- "Match": {"Type": "Callable", "Name": "Visit"},
- "Children": [
- {"Index": 0, "Name": "Input", "Type": "TExprBase"}
- ]
- },
- {
- "Name": "TCoTryWeakMemberFromDict",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "TryWeakMemberFromDict"},
- "Children": [
- {"Index": 0, "Name": "OtherDict", "Type": "TExprBase"},
- {"Index": 1, "Name": "RestDict", "Type": "TExprBase"},
- {"Index": 2, "Name": "Type", "Type": "TCoAtom"},
- {"Index": 3, "Name": "Name", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TCoMux",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "Mux"}
- },
- {
- "Name": "TCoDemux",
- "Base": "TCoInputBase",
- "Match": {"Type": "Callable", "Name": "Demux"}
- },
- {
- "Name": "TCoVariant",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "Variant"},
- "Children": [
- {"Index": 0, "Name": "Item", "Type": "TExprBase"},
- {"Index": 1, "Name": "Index", "Type": "TCoAtom"},
- {"Index": 2, "Name": "VarType", "Type": "TExprBase"}
- ]
- },
- {
- "Name": "TCoGuess",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "Guess"},
- "Children": [
- {"Index": 0, "Name": "Variant", "Type": "TExprBase"},
- {"Index": 1, "Name": "Index", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TCoSwitch",
- "Base": "TFreeArgCallable",
- "Match": {"Type": "Callable", "Name": "Switch"},
- "Children": [
- {"Index": 0, "Name": "Input", "Type": "TExprBase"},
- {"Index": 1, "Name": "BufferBytes", "Type": "TCoAtom"}
- ]
- },
- {
+ "Name": "TCoFromYsonSimpleType",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "FromYsonSimpleType"},
+ "Children": [
+ {"Index": 0, "Name": "Value", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Type", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TCoVisit",
+ "Base": "TFreeArgCallable",
+ "Match": {"Type": "Callable", "Name": "Visit"},
+ "Children": [
+ {"Index": 0, "Name": "Input", "Type": "TExprBase"}
+ ]
+ },
+ {
+ "Name": "TCoTryWeakMemberFromDict",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "TryWeakMemberFromDict"},
+ "Children": [
+ {"Index": 0, "Name": "OtherDict", "Type": "TExprBase"},
+ {"Index": 1, "Name": "RestDict", "Type": "TExprBase"},
+ {"Index": 2, "Name": "Type", "Type": "TCoAtom"},
+ {"Index": 3, "Name": "Name", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TCoMux",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "Mux"}
+ },
+ {
+ "Name": "TCoDemux",
+ "Base": "TCoInputBase",
+ "Match": {"Type": "Callable", "Name": "Demux"}
+ },
+ {
+ "Name": "TCoVariant",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "Variant"},
+ "Children": [
+ {"Index": 0, "Name": "Item", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Index", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "VarType", "Type": "TExprBase"}
+ ]
+ },
+ {
+ "Name": "TCoGuess",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "Guess"},
+ "Children": [
+ {"Index": 0, "Name": "Variant", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Index", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TCoSwitch",
+ "Base": "TFreeArgCallable",
+ "Match": {"Type": "Callable", "Name": "Switch"},
+ "Children": [
+ {"Index": 0, "Name": "Input", "Type": "TExprBase"},
+ {"Index": 1, "Name": "BufferBytes", "Type": "TCoAtom"}
+ ]
+ },
+ {
"Name": "TCoChopper",
"Base": "TCoInputBase",
"Match": {"Type": "Callable", "Name": "Chopper"},
@@ -1874,12 +1874,12 @@
]
},
{
- "Name": "TCoVariantItem",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "VariantItem"},
- "Children": [
- {"Index": 0, "Name": "Variant", "Type": "TExprBase"}
- ]
+ "Name": "TCoVariantItem",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "VariantItem"},
+ "Children": [
+ {"Index": 0, "Name": "Variant", "Type": "TExprBase"}
+ ]
},
{
"Name": "TCoEnsurePersistable",
@@ -1973,95 +1973,95 @@
"Children": [
{"Index": 0, "Name": "Range", "Type": "TExprBase"}
]
- },
- {
- "Name": "TCoCastStruct",
- "Base": "TCallable",
- "Match": {"Type": "Callable", "Name": "CastStruct"},
- "Children": [
- {"Index": 0, "Name": "Struct", "Type": "TExprBase"},
- {"Index": 1, "Name": "Type", "Type": "TExprBase"}
- ]
- },
- {
- "Name": "TCoNonDeterministicBase",
- "VarArgBase": "TExprBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"}
- },
- {
- "Name": "TCoRandomBase",
- "Base": "TCoNonDeterministicBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"}
- },
- {
- "Name": "TCoRandom",
- "Base": "TCoRandomBase",
- "Match": {"Type": "Callable", "Name": "Random"}
- },
- {
- "Name": "TCoRandomNumber",
- "Base": "TCoRandomBase",
- "Match": {"Type": "Callable", "Name": "RandomNumber"}
- },
- {
- "Name": "TCoRandomUuid",
- "Base": "TCoRandomBase",
- "Match": {"Type": "Callable", "Name": "RandomUuid"}
- },
- {
- "Name": "TCoTimeBase",
- "Base": "TCoNonDeterministicBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"}
- },
- {
- "Name": "TCoNow",
- "Base": "TCoTimeBase",
- "Match": {"Type": "Callable", "Name": "Now"}
- },
- {
- "Name": "TCoUtcTimeBase",
- "Base": "TCoTimeBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"}
- },
- {
- "Name": "TCoCurrentUtcDate",
- "Base": "TCoUtcTimeBase",
- "Match": {"Type": "Callable", "Name": "CurrentUtcDate"}
- },
- {
- "Name": "TCoCurrentUtcDatetime",
- "Base": "TCoUtcTimeBase",
- "Match": {"Type": "Callable", "Name": "CurrentUtcDatetime"}
- },
- {
- "Name": "TCoCurrentUtcTimestamp",
- "Base": "TCoUtcTimeBase",
- "Match": {"Type": "Callable", "Name": "CurrentUtcTimestamp"}
- },
- {
- "Name": "TCoTzTimeBase",
- "Base": "TCoTimeBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"}
- },
- {
- "Name": "TCoCurrentTzDate",
- "Base": "TCoTzTimeBase",
- "Match": {"Type": "Callable", "Name": "CurrentTzDate"}
- },
- {
- "Name": "TCoCurrentTzDatetime",
- "Base": "TCoTzTimeBase",
- "Match": {"Type": "Callable", "Name": "CurrentTzDatetime"}
- },
- {
- "Name": "TCoCurrentTzTimestamp",
- "Base": "TCoTzTimeBase",
- "Match": {"Type": "Callable", "Name": "CurrentTzTimestamp"}
+ },
+ {
+ "Name": "TCoCastStruct",
+ "Base": "TCallable",
+ "Match": {"Type": "Callable", "Name": "CastStruct"},
+ "Children": [
+ {"Index": 0, "Name": "Struct", "Type": "TExprBase"},
+ {"Index": 1, "Name": "Type", "Type": "TExprBase"}
+ ]
+ },
+ {
+ "Name": "TCoNonDeterministicBase",
+ "VarArgBase": "TExprBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"}
+ },
+ {
+ "Name": "TCoRandomBase",
+ "Base": "TCoNonDeterministicBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"}
+ },
+ {
+ "Name": "TCoRandom",
+ "Base": "TCoRandomBase",
+ "Match": {"Type": "Callable", "Name": "Random"}
+ },
+ {
+ "Name": "TCoRandomNumber",
+ "Base": "TCoRandomBase",
+ "Match": {"Type": "Callable", "Name": "RandomNumber"}
+ },
+ {
+ "Name": "TCoRandomUuid",
+ "Base": "TCoRandomBase",
+ "Match": {"Type": "Callable", "Name": "RandomUuid"}
+ },
+ {
+ "Name": "TCoTimeBase",
+ "Base": "TCoNonDeterministicBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"}
+ },
+ {
+ "Name": "TCoNow",
+ "Base": "TCoTimeBase",
+ "Match": {"Type": "Callable", "Name": "Now"}
+ },
+ {
+ "Name": "TCoUtcTimeBase",
+ "Base": "TCoTimeBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"}
+ },
+ {
+ "Name": "TCoCurrentUtcDate",
+ "Base": "TCoUtcTimeBase",
+ "Match": {"Type": "Callable", "Name": "CurrentUtcDate"}
+ },
+ {
+ "Name": "TCoCurrentUtcDatetime",
+ "Base": "TCoUtcTimeBase",
+ "Match": {"Type": "Callable", "Name": "CurrentUtcDatetime"}
+ },
+ {
+ "Name": "TCoCurrentUtcTimestamp",
+ "Base": "TCoUtcTimeBase",
+ "Match": {"Type": "Callable", "Name": "CurrentUtcTimestamp"}
+ },
+ {
+ "Name": "TCoTzTimeBase",
+ "Base": "TCoTimeBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"}
+ },
+ {
+ "Name": "TCoCurrentTzDate",
+ "Base": "TCoTzTimeBase",
+ "Match": {"Type": "Callable", "Name": "CurrentTzDate"}
+ },
+ {
+ "Name": "TCoCurrentTzDatetime",
+ "Base": "TCoTzTimeBase",
+ "Match": {"Type": "Callable", "Name": "CurrentTzDatetime"}
+ },
+ {
+ "Name": "TCoCurrentTzTimestamp",
+ "Base": "TCoTzTimeBase",
+ "Match": {"Type": "Callable", "Name": "CurrentTzTimestamp"}
}
]
}
diff --git a/ydb/library/yql/core/expr_nodes_gen/gen/__main__.py b/ydb/library/yql/core/expr_nodes_gen/gen/__main__.py
index 2bc809f4e7..eae903ae5d 100644
--- a/ydb/library/yql/core/expr_nodes_gen/gen/__main__.py
+++ b/ydb/library/yql/core/expr_nodes_gen/gen/__main__.py
@@ -38,11 +38,11 @@ for node in model["Nodes"]:
node["Builder"]["Kind"] = "List"
node["Builder"]["ListItemType"] = node["ListBase"]
- if "VarArgBase" in node:
- node["Base"] = "TVarArgCallable<{0}>".format(node["VarArgBase"])
- node["Builder"]["Kind"] = "List"
- node["Builder"]["ListItemType"] = node["VarArgBase"]
-
+ if "VarArgBase" in node:
+ node["Base"] = "TVarArgCallable<{0}>".format(node["VarArgBase"])
+ node["Builder"]["Kind"] = "List"
+ node["Builder"]["ListItemType"] = node["VarArgBase"]
+
if "Children" in node:
for child in node["Children"]:
childAux = child["aux"] = {}
diff --git a/ydb/library/yql/core/expr_nodes_gen/ya.make b/ydb/library/yql/core/expr_nodes_gen/ya.make
index b2ce04a289..ee11bf3687 100644
--- a/ydb/library/yql/core/expr_nodes_gen/ya.make
+++ b/ydb/library/yql/core/expr_nodes_gen/ya.make
@@ -1,10 +1,10 @@
LIBRARY()
-OWNER(
+OWNER(
spuchin
- g:yql
+ g:yql
g:yql_ydb_core
-)
+)
SRCS(
yql_expr_nodes_gen.h
diff --git a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h
index 0974e8d2d3..0defd1bac3 100644
--- a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h
+++ b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h
@@ -1,16 +1,16 @@
#pragma once
#include <ydb/library/yql/utils/yql_panic.h>
-
+
#include <ydb/library/yql/ast/yql_expr.h>
-#include <util/generic/vector.h>
-#include <util/generic/strbuf.h>
-#include <util/system/yassert.h>
+#include <util/generic/vector.h>
+#include <util/generic/strbuf.h>
+#include <util/system/yassert.h>
#include <util/string/cast.h>
-
+
#include <functional>
-#include <iterator>
+#include <iterator>
namespace NYql {
namespace NNodes {
@@ -91,10 +91,10 @@ public:
return *Raw_;
}
- const TExprNode* Raw() const {
- return Raw_;
- }
-
+ const TExprNode* Raw() const {
+ return Raw_;
+ }
+
bool IsValid() const {
return Raw_ != nullptr;
}
@@ -281,65 +281,65 @@ public:
}
};
-template<typename TItem>
-class TVarArgCallable : public TCallable {
-public:
- explicit TVarArgCallable(const TExprNode* node)
- : TCallable(node)
- {
- }
-
- explicit TVarArgCallable(const TExprNode::TPtr& node)
- : TCallable(node)
- {
- }
-
- TItem Arg(size_t index) const {
- return TItem(Ref().ChildPtr(index));
- }
- size_t ArgCount() const {
- return Ref().ChildrenSize();
- }
+template<typename TItem>
+class TVarArgCallable : public TCallable {
+public:
+ explicit TVarArgCallable(const TExprNode* node)
+ : TCallable(node)
+ {
+ }
+
+ explicit TVarArgCallable(const TExprNode::TPtr& node)
+ : TCallable(node)
+ {
+ }
+
+ TItem Arg(size_t index) const {
+ return TItem(Ref().ChildPtr(index));
+ }
+ size_t ArgCount() const {
+ return Ref().ChildrenSize();
+ }
TExprNode::TChildrenType Args() const {
- return Ref().Children();
- }
-
- TChildIterator<TItem> begin() const {
- return TChildIterator<TItem>(*this);
- }
- TChildIterator<TItem> end() const {
- return TChildIterator<TItem>();
- }
-};
-
-template<typename TItem>
-class TMaybeNode<TVarArgCallable<TItem>> : public TMaybeNode<TExprBase> {
-public:
- TMaybeNode(const TExprNode* node)
- : TMaybeNode<TExprBase>(node && TVarArgCallable<TItem>::Match(node) ? node : nullptr) {}
-
- TMaybeNode(const TExprNode::TPtr& node)
- : TMaybeNode<TExprBase>(node && TVarArgCallable<TItem>::Match(node.Get()) ? node : TExprNode::TPtr()) {}
-
- TVarArgCallable<TItem> Cast() const {
+ return Ref().Children();
+ }
+
+ TChildIterator<TItem> begin() const {
+ return TChildIterator<TItem>(*this);
+ }
+ TChildIterator<TItem> end() const {
+ return TChildIterator<TItem>();
+ }
+};
+
+template<typename TItem>
+class TMaybeNode<TVarArgCallable<TItem>> : public TMaybeNode<TExprBase> {
+public:
+ TMaybeNode(const TExprNode* node)
+ : TMaybeNode<TExprBase>(node && TVarArgCallable<TItem>::Match(node) ? node : nullptr) {}
+
+ TMaybeNode(const TExprNode::TPtr& node)
+ : TMaybeNode<TExprBase>(node && TVarArgCallable<TItem>::Match(node.Get()) ? node : TExprNode::TPtr()) {}
+
+ TVarArgCallable<TItem> Cast() const {
YQL_ENSURE(IsValid());
- return TMaybeNode<TExprBase>::Cast().template Cast<TVarArgCallable<TItem>>();
- }
-
- TMaybeNode<TItem> Arg(size_t index) const {
- if (!IsValid()) {
- return TMaybeNode<TItem>();
- }
-
- auto list = Cast();
- if (index >= list.ArgCount()) {
- return TMaybeNode<TItem>();
- }
-
- return TMaybeNode<TItem>(Ref().ChildPtr(index));
- }
-};
-
+ return TMaybeNode<TExprBase>::Cast().template Cast<TVarArgCallable<TItem>>();
+ }
+
+ TMaybeNode<TItem> Arg(size_t index) const {
+ if (!IsValid()) {
+ return TMaybeNode<TItem>();
+ }
+
+ auto list = Cast();
+ if (index >= list.ArgCount()) {
+ return TMaybeNode<TItem>();
+ }
+
+ return TMaybeNode<TItem>(Ref().ChildPtr(index));
+ }
+};
+
class TArgs
{
public:
@@ -443,21 +443,21 @@ public:
return parent.Value();
}
- TNodeBuilder<TParent, TDerived>& InitFrom(const ResultType& item) {
- Items.assign(item.begin(), item.end());
- return *static_cast<TNodeBuilder<TParent, TDerived>*>(this);
- }
-
+ TNodeBuilder<TParent, TDerived>& InitFrom(const ResultType& item) {
+ Items.assign(item.begin(), item.end());
+ return *static_cast<TNodeBuilder<TParent, TDerived>*>(this);
+ }
+
TNodeBuilder<TParent, TDerived>& Add(const TExprBase& node) {
Items.push_back(node);
return *static_cast<TNodeBuilder<TParent, TDerived>*>(this);
}
- TNodeBuilder<TParent, TDerived>& Add(const TExprNode::TPtr& node) {
- Items.push_back(TExprBase(node));
- return *static_cast<TNodeBuilder<TParent, TDerived>*>(this);
- }
-
+ TNodeBuilder<TParent, TDerived>& Add(const TExprNode::TPtr& node) {
+ Items.push_back(TExprBase(node));
+ return *static_cast<TNodeBuilder<TParent, TDerived>*>(this);
+ }
+
TNodeBuilder<TNodeBuilder<TParent, TDerived>, TItem> Add() {
return TNodeBuilder<TNodeBuilder<TParent, TDerived>, TItem>(this->Ctx, this->Pos,
[this] (const TExprBase& node) mutable -> TNodeBuilder<TParent, TDerived>& {
diff --git a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj
index 4b5733693f..cb995d3d8e 100644
--- a/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj
+++ b/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.jnj
@@ -4,7 +4,7 @@
#pragma once
#include <ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h>
-#include <util/generic/hash_set.h>
+#include <util/generic/hash_set.h>
namespace NYql {
namespace NNodes {
@@ -56,7 +56,7 @@ public:
{{child.Type}} {{child.Name}}() const {
return {{child.Type}}(this->Ref().ChildPtr(idx_{{child.Name}}));
}
- {% endif -%}
+ {% endif -%}
{%- endfor %}
{%- if node.Match %}
@@ -70,9 +70,9 @@ public:
static THashSet<TStringBuf> namesToMatch = {
{%- for name in node.aux.namesToMatch %}
TStringBuf("{{ name }}"),
- {%- endfor %}
- };
- return node->IsCallable(namesToMatch);
+ {%- endfor %}
+ };
+ return node->IsCallable(namesToMatch);
{% elif node.Match.Type == "Tuple" %}
return node->IsList()
&& node->ChildrenSize() >= {{ node.aux.fixedChildrenCount }}
@@ -99,7 +99,7 @@ protected:
public:
{% for child in node.Children %}
TMaybe<{{child.Type}}> {{child.Name}}() const {
- return TMaybe<{{child.Type}}>(this->IsValid() {% if child.Optional %}&& {{child.Index}} < this->Ref().ChildrenSize() {% endif %}? this->Ref().ChildPtr({{child.Index}}) : TExprNode::TPtr());
+ return TMaybe<{{child.Type}}>(this->IsValid() {% if child.Optional %}&& {{child.Index}} < this->Ref().ChildrenSize() {% endif %}? this->Ref().ChildPtr({{child.Index}}) : TExprNode::TPtr());
}
{% endfor %}
TDerived Cast() const {
@@ -118,20 +118,20 @@ protected:
, BuildFunc(buildFunc) {}
void Check() {
- {% if node.Match and node.Match.Type == "CallableBase" %}
- if (CallableNameHolder.empty()) {
- ythrow yexception() << "{{node.Name}} builder: CallableName not defined.";
- }
-
- static THashSet<TStringBuf> namesToMatch = {
- {%- for name in node.aux.namesToMatch %}
+ {% if node.Match and node.Match.Type == "CallableBase" %}
+ if (CallableNameHolder.empty()) {
+ ythrow yexception() << "{{node.Name}} builder: CallableName not defined.";
+ }
+
+ static THashSet<TStringBuf> namesToMatch = {
+ {%- for name in node.aux.namesToMatch %}
TStringBuf("{{ name }}"),
- {%- endfor %}
- };
- if (namesToMatch.find(CallableNameHolder) == namesToMatch.end()) {
- ythrow yexception() << "{{node.Name}} builder: Invalid CallableName: " << CallableNameHolder;
- }
- {% endif %}
+ {%- endfor %}
+ };
+ if (namesToMatch.find(CallableNameHolder) == namesToMatch.end()) {
+ ythrow yexception() << "{{node.Name}} builder: Invalid CallableName: " << CallableNameHolder;
+ }
+ {% endif %}
{% if node.aux.allChildren | count > 0 %}
bool argsValid = true;
bool argValid;
@@ -151,9 +151,9 @@ protected:
{% endif %}
}
- {% if node.Match and node.Match.Type == "CallableBase" %}
- TString CallableNameHolder;
- {% endif %}
+ {% if node.Match and node.Match.Type == "CallableBase" %}
+ TString CallableNameHolder;
+ {% endif %}
{% for child in node.aux.allChildren %}
TMaybe<{{child.Type}}> {{child.aux.holderName}};
{%- endfor %}
@@ -174,33 +174,33 @@ public:
return parent.Value();
}
- {% if node.aux.allChildren | count > 0 %}
- TNodeBuilder<TParent, TDerived>& InitFrom(const ResultType& item) {
- {% if node.Match.Type == "CallableBase" %}
- CallableNameHolder = item.Ref().Content();
- {% endif %}
- {%- for child in node.aux.allChildren %}
- {%- if child.Optional %}
- if (item.{{child.Name}}()) {
- {{child.aux.holderName}} = item.{{child.Name}}().Cast();
- }
- {%- else %}
- {{child.aux.holderName}} = item.{{child.Name}}();
- {%- endif %}
- {%- endfor %}
-
- {%- if node.Builder.Kind == "FreeArg" %}
- TBase::FreeArgsHolder.assign(item.FreeArgs().begin(), item.FreeArgs().end());
- {%- endif %}
-
- return *static_cast<TNodeBuilder<TParent, TDerived>*>(this);
- }
- {% endif %}
-
- {% if node.Match and node.Match.Type == "CallableBase" %}
- TBuilder<TParent, TDerived>& CallableName(TStringBuf name) { CallableNameHolder = name; return *static_cast<TBuilder<TParent, TDerived>*>(this); }
- {% endif %}
-
+ {% if node.aux.allChildren | count > 0 %}
+ TNodeBuilder<TParent, TDerived>& InitFrom(const ResultType& item) {
+ {% if node.Match.Type == "CallableBase" %}
+ CallableNameHolder = item.Ref().Content();
+ {% endif %}
+ {%- for child in node.aux.allChildren %}
+ {%- if child.Optional %}
+ if (item.{{child.Name}}()) {
+ {{child.aux.holderName}} = item.{{child.Name}}().Cast();
+ }
+ {%- else %}
+ {{child.aux.holderName}} = item.{{child.Name}}();
+ {%- endif %}
+ {%- endfor %}
+
+ {%- if node.Builder.Kind == "FreeArg" %}
+ TBase::FreeArgsHolder.assign(item.FreeArgs().begin(), item.FreeArgs().end());
+ {%- endif %}
+
+ return *static_cast<TNodeBuilder<TParent, TDerived>*>(this);
+ }
+ {% endif %}
+
+ {% if node.Match and node.Match.Type == "CallableBase" %}
+ TBuilder<TParent, TDerived>& CallableName(TStringBuf name) { CallableNameHolder = name; return *static_cast<TBuilder<TParent, TDerived>*>(this); }
+ {% endif %}
+
{%- for child in node.aux.allChildren %}
TBuilder<TParent, TDerived>& {{child.Name}}(const TStringBuf& argName) { {{child.aux.holderName}} = {{child.Type}}(this->GetArgFunc(argName)); return *static_cast<TBuilder<TParent, TDerived>*>(this); }
{% if child.Optional %}
@@ -210,13 +210,13 @@ public:
TBuilder<TParent, TDerived>& {{child.Name}}(const TExprNode::TPtr& node) { {{child.aux.holderName}} = {{child.Type}}(node); return *static_cast<TBuilder<TParent, TDerived>*>(this); }
- TBuilder<TParent, TDerived>& {{child.Name}}(std::function<void(TExprNodeBuilder&)> fluentFunc) {
+ TBuilder<TParent, TDerived>& {{child.Name}}(std::function<void(TExprNodeBuilder&)> fluentFunc) {
TExprNodeBuilder fluentBuilder(this->Pos, this->Ctx, [this] (const TStringBuf& argName) {
- return this->GetArgFunc(argName).Ptr();
+ return this->GetArgFunc(argName).Ptr();
});
- fluentFunc(fluentBuilder);
- {{child.aux.holderName}} = fluentBuilder.Build();
+ fluentFunc(fluentBuilder);
+ {{child.aux.holderName}} = fluentBuilder.Build();
return *static_cast<TBuilder<TParent, TDerived>*>(this);
}
@@ -321,7 +321,7 @@ class UnknownBuilder {
static_assert(false, "Unknown builder kind: {{ node.Builder.Kind }}.");
{% endif %}
{{ node.Name }} DoBuild() {
- {%- if node.Match and (node.Match.Type == "Callable" or node.Match.Type == "CallableBase") %}
+ {%- if node.Match and (node.Match.Type == "Callable" or node.Match.Type == "CallableBase") %}
TExprNode::TListType argsList;
{%- if node.Builder.Kind == "List" %}
for (auto child : this->Items) {
@@ -342,11 +342,11 @@ class UnknownBuilder {
}
{% endif %}
{%- endif %}
- {%- if node.Match.Type == "CallableBase" %}
- auto node = this->Ctx.NewCallable(this->Pos, this->CallableNameHolder, std::move(argsList));
- {%- else %}
+ {%- if node.Match.Type == "CallableBase" %}
+ auto node = this->Ctx.NewCallable(this->Pos, this->CallableNameHolder, std::move(argsList));
+ {%- else %}
auto node = this->Ctx.NewCallable(this->Pos, "{{ node.Match.Name }}", std::move(argsList));
- {%- endif %}
+ {%- endif %}
return {{ node.Name }}(node);
{%- elif node.Match and node.Match.Type == "Tuple"%}
TExprNode::TListType tupleItems;
diff --git a/ydb/library/yql/core/extract_predicate/extract_predicate_impl.cpp b/ydb/library/yql/core/extract_predicate/extract_predicate_impl.cpp
index e2c16395b7..dd85781565 100644
--- a/ydb/library/yql/core/extract_predicate/extract_predicate_impl.cpp
+++ b/ydb/library/yql/core/extract_predicate/extract_predicate_impl.cpp
@@ -715,8 +715,8 @@ THolder<IGraphTransformer> CreateRangeExtractionOptimizer(TTypeAnnotationContext
auto issueCode = TIssuesIds::CORE_EXEC;
pipeline.AddServiceTransformers(issueCode);
pipeline.AddTypeAnnotationTransformer(issueCode);
- // pipeline.Add(TExprLogTransformer::Sync("ExtractPredicateOpt", NLog::EComponent::Core, NLog::ELevel::TRACE),
- // "ExtractPredicateOpt", issueCode, "ExtractPredicateOpt");
+ // pipeline.Add(TExprLogTransformer::Sync("ExtractPredicateOpt", NLog::EComponent::Core, NLog::ELevel::TRACE),
+ // "ExtractPredicateOpt", issueCode, "ExtractPredicateOpt");
pipeline.Add(CreateFunctorTransformer(
[](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
DoOptimizeForRangeExtraction(input, output, true, ctx);
diff --git a/ydb/library/yql/core/facade/yql_facade.cpp b/ydb/library/yql/core/facade/yql_facade.cpp
index b74abc1c34..0918e9d3a2 100644
--- a/ydb/library/yql/core/facade/yql_facade.cpp
+++ b/ydb/library/yql/core/facade/yql_facade.cpp
@@ -24,7 +24,7 @@
#include <ydb/library/yql/public/issue/yql_issue.h>
#include <library/cpp/yson/node/node_io.h>
-#include <library/cpp/deprecated/split/split_iterator.h>
+#include <library/cpp/deprecated/split/split_iterator.h>
#include <library/cpp/yson/writer.h>
#include <library/cpp/string_utils/base64/base64.h>
@@ -32,8 +32,8 @@
#include <util/stream/null.h>
#include <util/string/split.h>
#include <util/generic/guid.h>
-#include <util/system/rusage.h>
-#include <util/generic/yexception.h>
+#include <util/system/rusage.h>
+#include <util/generic/yexception.h>
using namespace NThreading;
@@ -116,13 +116,13 @@ TProgramFactory::TProgramFactory(
const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
ui64 nextUniqueId,
const TVector<TDataProviderInitializer>& dataProvidersInit,
- const TString& runner)
+ const TString& runner)
: UseRepeatableRandomAndTimeProviders_(useRepeatableRandomAndTimeProviders)
, FunctionRegistry_(functionRegistry)
, NextUniqueId_(nextUniqueId)
, DataProvidersInit_(dataProvidersInit)
, GatewaysConfig_(nullptr)
- , Runner_(runner)
+ , Runner_(runner)
{
AddCredentialsTable(std::make_shared<TCredentialTable>());
}
@@ -159,10 +159,10 @@ void TProgramFactory::SetModules(IModuleResolver::TPtr modules) {
Modules_ = modules;
}
-void TProgramFactory::SetUdfResolver(IUdfResolver::TPtr udfResolver) {
- UdfResolver_ = udfResolver;
-}
-
+void TProgramFactory::SetUdfResolver(IUdfResolver::TPtr udfResolver) {
+ UdfResolver_ = udfResolver;
+}
+
void TProgramFactory::SetUdfIndex(TUdfIndex::TPtr udfIndex, TUdfIndexPackageSet::TPtr udfIndexPackageSet) {
UdfIndex_ = std::move(udfIndex);
UdfIndexPackageSet_ = std::move(udfIndexPackageSet);
@@ -214,14 +214,14 @@ TProgram::TProgram(
const TVector<TCredentialTablePtr>& credentialTables,
const TUserCredentials& userCredentials,
const IModuleResolver::TPtr& modules,
- const IUdfResolver::TPtr& udfResolver,
+ const IUdfResolver::TPtr& udfResolver,
const TUdfIndex::TPtr& udfIndex,
const TUdfIndexPackageSet::TPtr& udfIndexPackageSet,
const TFileStoragePtr& fileStorage,
const TGatewaysConfig* gatewaysConfig,
const TString& filename,
const TString& sourceCode,
- const TString& sessionId,
+ const TString& sessionId,
const TString& runner,
bool enableRangeComputeFor
)
@@ -232,7 +232,7 @@ TProgram::TProgram(
, DataProvidersInit_(dataProvidersInit)
, CredentialTables_(credentialTables)
, UserCredentials_(userCredentials)
- , UdfResolver_(udfResolver)
+ , UdfResolver_(udfResolver)
, UdfIndex_(udfIndex)
, UdfIndexPackageSet_(udfIndexPackageSet)
, FileStorage_(fileStorage)
@@ -258,7 +258,7 @@ TProgram::TProgram(
modules->AttachUserData(UserDataStorage_);
modules->SetUrlLoader(new TUrlLoader(FileStorage_));
}
- OperationOptions_.Runner = runner;
+ OperationOptions_.Runner = runner;
}
TProgram::~TProgram() {
@@ -286,11 +286,11 @@ void TProgram::SetDisableNativeUdfSupport(bool disable) {
DisableNativeUdfSupport_ = disable;
}
-void TProgram::SetUseTableMetaFromGraph(bool use) {
- Y_ENSURE(!TypeCtx_, "TypeCtx_ already created");
- UseTableMetaFromGraph_ = use;
-}
-
+void TProgram::SetUseTableMetaFromGraph(bool use) {
+ Y_ENSURE(!TypeCtx_, "TypeCtx_ already created");
+ UseTableMetaFromGraph_ = use;
+}
+
TTypeAnnotationContextPtr TProgram::GetAnnotationContext() const {
Y_ENSURE(TypeCtx_, "TypeCtx_ is not created");
return TypeCtx_;
@@ -302,7 +302,7 @@ TTypeAnnotationContextPtr TProgram::ProvideAnnotationContext(const TString& user
TypeCtx_->OperationOptions = OperationOptions_;
TypeCtx_->ValidateMode = ValidateMode_;
TypeCtx_->DisableNativeUdfSupport = DisableNativeUdfSupport_;
- TypeCtx_->UseTableMetaFromGraph = UseTableMetaFromGraph_;
+ TypeCtx_->UseTableMetaFromGraph = UseTableMetaFromGraph_;
}
return TypeCtx_;
@@ -399,7 +399,7 @@ bool TProgram::ParseSql() {
YQL_PROFILE_FUNC(TRACE);
static const THashMap<TString, TString> clusters = {
- { "plato", TString(YtProviderName) }
+ { "plato", TString(YtProviderName) }
};
NSQLTranslation::TTranslationSettings settings;
@@ -459,9 +459,9 @@ bool TProgram::CollectUsedClusters() {
bool hasErrors = false;
VisitExpr(ExprRoot_, [&typesCtx, &ctx, &usedClusters, &usedProviders, &hasErrors](const TExprNode::TPtr& node) {
- if (auto dsNode = TMaybeNode<TCoDataSource>(node)) {
- auto datasource = typesCtx.DataSourceMap.FindPtr(dsNode.Cast().Category());
- YQL_ENSURE(datasource, "Unknown DataSource: " << dsNode.Cast().Category().Value());
+ if (auto dsNode = TMaybeNode<TCoDataSource>(node)) {
+ auto datasource = typesCtx.DataSourceMap.FindPtr(dsNode.Cast().Category());
+ YQL_ENSURE(datasource, "Unknown DataSource: " << dsNode.Cast().Category().Value());
TMaybe<TString> cluster;
if (!(*datasource)->ValidateParameters(*node, ctx, cluster)) {
@@ -475,9 +475,9 @@ bool TProgram::CollectUsedClusters() {
}
}
- if (auto dsNode = TMaybeNode<TCoDataSink>(node)) {
- auto datasink = typesCtx.DataSinkMap.FindPtr(dsNode.Cast().Category());
- YQL_ENSURE(datasink, "Unknown DataSink: " << dsNode.Cast().Category().Value());
+ if (auto dsNode = TMaybeNode<TCoDataSink>(node)) {
+ auto datasink = typesCtx.DataSinkMap.FindPtr(dsNode.Cast().Category());
+ YQL_ENSURE(datasink, "Unknown DataSink: " << dsNode.Cast().Category().Value());
TMaybe<TString> cluster;
if (!(*datasink)->ValidateParameters(*node, ctx, cluster)) {
@@ -504,46 +504,46 @@ bool TProgram::CollectUsedClusters() {
return true;
}
-TProgram::TStatus TProgram::Discover(const TString& username) {
- YQL_PROFILE_FUNC(TRACE);
- auto m = &TProgram::DiscoverAsync;
- return SyncExecution(this, m, username);
-}
-
-TProgram::TFutureStatus TProgram::DiscoverAsync(const TString& username) {
- if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) {
- return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
- }
- TypeCtx_->DiscoveryMode = true;
-
- Y_ENSURE(ExprRoot_, "Program not compiled yet");
-
- Transformer_ = TTransformationPipeline(TypeCtx_)
- .AddServiceTransformers()
- .AddParametersEvaluation(*FunctionRegistry_)
- .AddPreTypeAnnotation()
- .AddExpressionEvaluation(*FunctionRegistry_)
- .AddPreIOAnnotation()
- .Build();
-
- TFuture<void> openSession = OpenSession(username);
- if (!openSession.Initialized()) {
- return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
- }
-
- return openSession.Apply([this](const TFuture<void>& f) {
- YQL_LOG_CTX_ROOT_SCOPE(GetSessionId());
- try {
- f.GetValue();
- } catch (const std::exception& e) {
- YQL_LOG(ERROR) << "OpenSession error: " << e.what();
- ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
- return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
- }
- return AsyncTransform(*Transformer_, ExprRoot_, *ExprCtx_, false);
- });
-}
-
+TProgram::TStatus TProgram::Discover(const TString& username) {
+ YQL_PROFILE_FUNC(TRACE);
+ auto m = &TProgram::DiscoverAsync;
+ return SyncExecution(this, m, username);
+}
+
+TProgram::TFutureStatus TProgram::DiscoverAsync(const TString& username) {
+ if (!ProvideAnnotationContext(username)->Initialize(*ExprCtx_) || !CollectUsedClusters()) {
+ return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
+ }
+ TypeCtx_->DiscoveryMode = true;
+
+ Y_ENSURE(ExprRoot_, "Program not compiled yet");
+
+ Transformer_ = TTransformationPipeline(TypeCtx_)
+ .AddServiceTransformers()
+ .AddParametersEvaluation(*FunctionRegistry_)
+ .AddPreTypeAnnotation()
+ .AddExpressionEvaluation(*FunctionRegistry_)
+ .AddPreIOAnnotation()
+ .Build();
+
+ TFuture<void> openSession = OpenSession(username);
+ if (!openSession.Initialized()) {
+ return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
+ }
+
+ return openSession.Apply([this](const TFuture<void>& f) {
+ YQL_LOG_CTX_ROOT_SCOPE(GetSessionId());
+ try {
+ f.GetValue();
+ } catch (const std::exception& e) {
+ YQL_LOG(ERROR) << "OpenSession error: " << e.what();
+ ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
+ return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
+ }
+ return AsyncTransform(*Transformer_, ExprRoot_, *ExprCtx_, false);
+ });
+}
+
TProgram::TStatus TProgram::Validate(const TString& username, IOutputStream* exprOut, bool withTypes) {
YQL_PROFILE_FUNC(TRACE);
auto m = &TProgram::ValidateAsync;
@@ -572,10 +572,10 @@ TProgram::TFutureStatus TProgram::ValidateAsync(const TString& username, IOutput
Transformer_ = TTransformationPipeline(TypeCtx_)
.AddServiceTransformers()
.AddParametersEvaluation(*FunctionRegistry_)
- .AddPreTypeAnnotation()
- .AddExpressionEvaluation(*FunctionRegistry_)
- .AddIOAnnotation()
- .AddTypeAnnotation()
+ .AddPreTypeAnnotation()
+ .AddExpressionEvaluation(*FunctionRegistry_)
+ .AddIOAnnotation()
+ .AddTypeAnnotation()
.Add(TExprOutputTransformer::Sync(ExprRoot_, exprOut, withTypes), "AstOutput")
.Build();
@@ -584,16 +584,16 @@ TProgram::TFutureStatus TProgram::ValidateAsync(const TString& username, IOutput
return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
}
- return openSession.Apply([this](const TFuture<void>& f) {
+ return openSession.Apply([this](const TFuture<void>& f) {
YQL_LOG_CTX_ROOT_SCOPE(GetSessionId());
- try {
- f.GetValue();
- } catch (const std::exception& e) {
- YQL_LOG(ERROR) << "OpenSession error: " << e.what();
- ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
- return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
- }
- return AsyncTransformWithFallback(false);
+ try {
+ f.GetValue();
+ } catch (const std::exception& e) {
+ YQL_LOG(ERROR) << "OpenSession error: " << e.what();
+ ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
+ return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
+ }
+ return AsyncTransformWithFallback(false);
});
}
@@ -640,31 +640,31 @@ TProgram::TFutureStatus TProgram::OptimizeAsync(
Transformer_ = TTransformationPipeline(TypeCtx_)
.AddServiceTransformers()
.AddParametersEvaluation(*FunctionRegistry_)
- .AddPreTypeAnnotation()
- .AddExpressionEvaluation(*FunctionRegistry_)
- .AddIOAnnotation()
- .AddTypeAnnotation()
- .AddPostTypeAnnotation()
+ .AddPreTypeAnnotation()
+ .AddExpressionEvaluation(*FunctionRegistry_)
+ .AddIOAnnotation()
+ .AddTypeAnnotation()
+ .AddPostTypeAnnotation()
.Add(TExprOutputTransformer::Sync(ExprRoot_, traceOut), "ExprOutput")
.AddOptimization()
.Add(CreatePlanInfoTransformer(*TypeCtx_), "PlanInfo")
.Add(TExprOutputTransformer::Sync(ExprRoot_, exprOut, withTypes), "AstOutput")
- .Add(TPlanOutputTransformer::Sync(tracePlan, GetPlanBuilder(), OutputFormat_), "PlanOutput")
+ .Add(TPlanOutputTransformer::Sync(tracePlan, GetPlanBuilder(), OutputFormat_), "PlanOutput")
.Build();
TFuture<void> openSession = OpenSession(username);
if (!openSession.Initialized())
return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
- return openSession.Apply([this](const TFuture<void>& f) {
+ return openSession.Apply([this](const TFuture<void>& f) {
YQL_LOG_CTX_ROOT_SCOPE(GetSessionId());
- try {
- f.GetValue();
- } catch (const std::exception& e) {
- YQL_LOG(ERROR) << "OpenSession error: " << e.what();
- ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
- return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
- }
+ try {
+ f.GetValue();
+ } catch (const std::exception& e) {
+ YQL_LOG(ERROR) << "OpenSession error: " << e.what();
+ ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
+ return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
+ }
return AsyncTransformWithFallback(false);
});
}
@@ -703,11 +703,11 @@ TProgram::TFutureStatus TProgram::OptimizeAsyncWithConfig(
pipelineConf.AfterCreate(&pipeline);
pipeline.AddServiceTransformers();
pipeline.AddParametersEvaluation(*FunctionRegistry_);
- pipeline.AddPreTypeAnnotation();
- pipeline.AddExpressionEvaluation(*FunctionRegistry_);
- pipeline.AddIOAnnotation();
- pipeline.AddTypeAnnotation();
- pipeline.AddPostTypeAnnotation();
+ pipeline.AddPreTypeAnnotation();
+ pipeline.AddExpressionEvaluation(*FunctionRegistry_);
+ pipeline.AddIOAnnotation();
+ pipeline.AddTypeAnnotation();
+ pipeline.AddPostTypeAnnotation();
pipelineConf.AfterTypeAnnotation(&pipeline);
pipeline.AddOptimization();
@@ -725,15 +725,15 @@ TProgram::TFutureStatus TProgram::OptimizeAsyncWithConfig(
if (!openSession.Initialized())
return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
- return openSession.Apply([this](const TFuture<void>& f) {
+ return openSession.Apply([this](const TFuture<void>& f) {
YQL_LOG_CTX_ROOT_SCOPE(GetSessionId());
- try {
- f.GetValue();
- } catch (const std::exception& e) {
- YQL_LOG(ERROR) << "OpenSession error: " << e.what();
- ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
- return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
- }
+ try {
+ f.GetValue();
+ } catch (const std::exception& e) {
+ YQL_LOG(ERROR) << "OpenSession error: " << e.what();
+ ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
+ return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
+ }
return AsyncTransformWithFallback(false);
});
}
@@ -807,15 +807,15 @@ TProgram::TFutureStatus TProgram::RunAsync(
SavedExprRoot_ = ExprRoot_;
- return openSession.Apply([this](const TFuture<void>& f) {
+ return openSession.Apply([this](const TFuture<void>& f) {
YQL_LOG_CTX_ROOT_SCOPE(GetSessionId());
- try {
- f.GetValue();
- } catch (const std::exception& e) {
- YQL_LOG(ERROR) << "OpenSession error: " << e.what();
- ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
- return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
- }
+ try {
+ f.GetValue();
+ } catch (const std::exception& e) {
+ YQL_LOG(ERROR) << "OpenSession error: " << e.what();
+ ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
+ return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
+ }
return AsyncTransformWithFallback(false);
});
}
@@ -855,11 +855,11 @@ TProgram::TFutureStatus TProgram::RunAsyncWithConfig(
pipelineConf.AfterCreate(&pipeline);
pipeline.AddServiceTransformers();
pipeline.AddParametersEvaluation(*FunctionRegistry_);
- pipeline.AddPreTypeAnnotation();
- pipeline.AddExpressionEvaluation(*FunctionRegistry_);
- pipeline.AddIOAnnotation();
- pipeline.AddTypeAnnotation();
- pipeline.AddPostTypeAnnotation();
+ pipeline.AddPreTypeAnnotation();
+ pipeline.AddExpressionEvaluation(*FunctionRegistry_);
+ pipeline.AddIOAnnotation();
+ pipeline.AddTypeAnnotation();
+ pipeline.AddPostTypeAnnotation();
pipelineConf.AfterTypeAnnotation(&pipeline);
pipeline.AddOptimization();
@@ -868,7 +868,7 @@ TProgram::TFutureStatus TProgram::RunAsyncWithConfig(
"ExpandRangeComputeFor", TIssuesIds::CORE_EXEC);
}
pipelineConf.AfterOptimize(&pipeline);
- pipeline.AddRun(ProgressWriter_);
+ pipeline.AddRun(ProgressWriter_);
Transformer_ = pipeline.Build();
@@ -879,15 +879,15 @@ TProgram::TFutureStatus TProgram::RunAsyncWithConfig(
SavedExprRoot_ = ExprRoot_;
- return openSession.Apply([this](const TFuture<void>& f) {
+ return openSession.Apply([this](const TFuture<void>& f) {
YQL_LOG_CTX_ROOT_SCOPE(GetSessionId());
- try {
- f.GetValue();
- } catch (const std::exception& e) {
- YQL_LOG(ERROR) << "OpenSession error: " << e.what();
- ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
- return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
- }
+ try {
+ f.GetValue();
+ } catch (const std::exception& e) {
+ YQL_LOG(ERROR) << "OpenSession error: " << e.what();
+ ExprCtx_->IssueManager.RaiseIssue(ExceptionToIssue(e));
+ return NThreading::MakeFuture<TStatus>(IGraphTransformer::TStatus::Error);
+ }
return AsyncTransformWithFallback(false);
});
}
@@ -966,7 +966,7 @@ TFuture<IGraphTransformer::TStatus> TProgram::AsyncTransformWithFallback(bool ap
for (auto source : TypeCtx_->DataSources) {
source->Reset();
}
- CleanupLastSession();
+ CleanupLastSession();
if (hasDqGatewayError) {
TIssue warning("DQ cannot execute the query");
@@ -1066,8 +1066,8 @@ TMaybe<TString> TProgram::GetDiagnostics() {
NCommon::TransformerStatsToYson("", transformStats, writer);
for (auto& datasink : TypeCtx_->DataSinks) {
- writer.OnKeyedItem(datasink->GetName());
- if (!datasink->CollectDiagnostics(writer)) {
+ writer.OnKeyedItem(datasink->GetName());
+ if (!datasink->CollectDiagnostics(writer)) {
writer.OnEntity();
}
}
@@ -1081,10 +1081,10 @@ TMaybe<TString> TProgram::GetDiagnostics() {
return out.Str();
}
-IGraphTransformer::TStatistics TProgram::GetRawDiagnostics() {
- return Transformer_ ? Transformer_->GetStatistics() : IGraphTransformer::TStatistics::NotPresent();
-}
-
+IGraphTransformer::TStatistics TProgram::GetRawDiagnostics() {
+ return Transformer_ ? Transformer_->GetStatistics() : IGraphTransformer::TStatistics::NotPresent();
+}
+
TMaybe<TString> TProgram::GetTasksInfo() {
if (!TypeCtx_) {
return Nothing();
@@ -1104,7 +1104,7 @@ TMaybe<TString> TProgram::GetTasksInfo() {
writer.OnBeginList();
for (auto& datasink : TypeCtx_->DataSinks) {
- hasTasks = hasTasks || datasink->GetTasksInfo(writer);
+ hasTasks = hasTasks || datasink->GetTasksInfo(writer);
}
writer.OnEndList();
@@ -1119,74 +1119,74 @@ TMaybe<TString> TProgram::GetTasksInfo() {
}
}
-TMaybe<TString> TProgram::GetStatistics(bool totalOnly) {
+TMaybe<TString> TProgram::GetStatistics(bool totalOnly) {
if (!TypeCtx_) {
return Nothing();
}
TStringStream out;
NYson::TYsonWriter writer(&out);
- // Header
+ // Header
writer.OnBeginMap();
writer.OnKeyedItem("ExecutionStatistics");
writer.OnBeginMap();
- // Providers
+ // Providers
bool hasStatistics = false;
for (auto& datasink : TypeCtx_->DataSinks) {
- TStringStream providerOut;
+ TStringStream providerOut;
NYson::TYsonWriter providerWriter(&providerOut);
- if (datasink->CollectStatistics(providerWriter, totalOnly)) {
+ if (datasink->CollectStatistics(providerWriter, totalOnly)) {
writer.OnKeyedItem(datasink->GetName());
- writer.OnRaw(providerOut.Str());
- hasStatistics = true;
+ writer.OnRaw(providerOut.Str());
+ hasStatistics = true;
}
}
- auto rusage = TRusage::Get();
- // System stats
- writer.OnKeyedItem("system");
- writer.OnBeginMap();
- writer.OnKeyedItem("MaxRSS");
- writer.OnBeginMap();
- writer.OnKeyedItem("max");
- writer.OnInt64Scalar(rusage.MaxRss);
- writer.OnEndMap();
-
- writer.OnKeyedItem("MajorPageFaults");
- writer.OnBeginMap();
- writer.OnKeyedItem("count");
- writer.OnInt64Scalar(rusage.MajorPageFaults);
- writer.OnEndMap();
- writer.OnEndMap(); // system
-
- // Footer
+ auto rusage = TRusage::Get();
+ // System stats
+ writer.OnKeyedItem("system");
+ writer.OnBeginMap();
+ writer.OnKeyedItem("MaxRSS");
+ writer.OnBeginMap();
+ writer.OnKeyedItem("max");
+ writer.OnInt64Scalar(rusage.MaxRss);
+ writer.OnEndMap();
+
+ writer.OnKeyedItem("MajorPageFaults");
+ writer.OnBeginMap();
+ writer.OnKeyedItem("count");
+ writer.OnInt64Scalar(rusage.MajorPageFaults);
+ writer.OnEndMap();
+ writer.OnEndMap(); // system
+
+ // Footer
writer.OnEndMap();
writer.OnEndMap();
if (hasStatistics) {
return out.Str();
}
return Nothing();
-}
-
-TMaybe<TString> TProgram::GetDiscoveredData() {
- if (!TypeCtx_) {
- return Nothing();
- }
-
- TStringStream out;
+}
+
+TMaybe<TString> TProgram::GetDiscoveredData() {
+ if (!TypeCtx_) {
+ return Nothing();
+ }
+
+ TStringStream out;
NYson::TYsonWriter writer(&out);
- writer.OnBeginMap();
- for (auto& datasource: TypeCtx_->DataSources) {
- TStringStream providerOut;
+ writer.OnBeginMap();
+ for (auto& datasource: TypeCtx_->DataSources) {
+ TStringStream providerOut;
NYson::TYsonWriter providerWriter(&providerOut);
- if (datasource->CollectDiscoveredData(providerWriter)) {
- writer.OnKeyedItem(datasource->GetName());
- writer.OnRaw(providerOut.Str());
- }
- }
- writer.OnEndMap();
- return out.Str();
+ if (datasource->CollectDiscoveredData(providerWriter)) {
+ writer.OnKeyedItem(datasource->GetName());
+ writer.OnRaw(providerOut.Str());
+ }
+ }
+ writer.OnEndMap();
+ return out.Str();
}
TProgram::TFutureStatus TProgram::ContinueAsync() {
@@ -1199,21 +1199,21 @@ void TProgram::Abort()
CloseLastSession();
}
-void TProgram::CleanupLastSession() {
- YQL_LOG_CTX_ROOT_SCOPE(GetSessionId());
-
+void TProgram::CleanupLastSession() {
+ YQL_LOG_CTX_ROOT_SCOPE(GetSessionId());
+
TString sessionId = GetSessionId();
- if (sessionId.empty()) {
- return;
- }
-
- for (const auto& dp : DataProviders_) {
- if (dp.CleanupSession) {
- dp.CleanupSession(sessionId);
- }
- }
-}
-
+ if (sessionId.empty()) {
+ return;
+ }
+
+ for (const auto& dp : DataProviders_) {
+ if (dp.CleanupSession) {
+ dp.CleanupSession(sessionId);
+ }
+ }
+}
+
void TProgram::CloseLastSession() {
YQL_LOG_CTX_ROOT_SCOPE(GetSessionId());
@@ -1251,7 +1251,7 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us
typeAnnotationContext->Credentials = CredentialTables_;
typeAnnotationContext->UserCredentials = UserCredentials_;
typeAnnotationContext->Modules = Modules_;
- typeAnnotationContext->UdfResolver = UdfResolver_;
+ typeAnnotationContext->UdfResolver = UdfResolver_;
typeAnnotationContext->UdfIndex = UdfIndex_;
typeAnnotationContext->UdfIndexPackageSet = UdfIndexPackageSet_;
typeAnnotationContext->RandomProvider = RandomProvider_;
@@ -1268,7 +1268,7 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us
PlanBuilder_ = CreatePlanBuilder(*typeAnnotationContext);
THashSet<TString> providerNames;
- TVector<TString> fullResultDataSinks;
+ TVector<TString> fullResultDataSinks;
TVector<std::function<TMaybe<TString>(const TString& url)>> tokenResolvers;
for (const auto& dpi : DataProvidersInit_) {
auto dp = dpi(
@@ -1282,34 +1282,34 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us
OperationOptions_
);
- providerNames.insert(dp.Names.begin(), dp.Names.end());
+ providerNames.insert(dp.Names.begin(), dp.Names.end());
DataProviders_.emplace_back(dp);
if (dp.Source) {
- typeAnnotationContext->AddDataSource(dp.Names, dp.Source);
+ typeAnnotationContext->AddDataSource(dp.Names, dp.Source);
}
if (dp.Sink) {
- typeAnnotationContext->AddDataSink(dp.Names, dp.Sink);
- }
+ typeAnnotationContext->AddDataSink(dp.Names, dp.Sink);
+ }
if (dp.TokenResolver) {
tokenResolvers.push_back(dp.TokenResolver);
}
-
- if (dp.SupportFullResultDataSink) {
- fullResultDataSinks.insert(fullResultDataSinks.end(), dp.Names.begin(), dp.Names.end());
- }
+
+ if (dp.SupportFullResultDataSink) {
+ fullResultDataSinks.insert(fullResultDataSinks.end(), dp.Names.begin(), dp.Names.end());
+ }
}
TVector<TString> resultProviderDataSources;
if (providerNames.contains(YtProviderName)) {
- resultProviderDataSources.push_back(TString(YtProviderName));
- }
-
+ resultProviderDataSources.push_back(TString(YtProviderName));
+ }
+
if (providerNames.contains(KikimrProviderName)) {
resultProviderDataSources.push_back(TString(KikimrProviderName));
- }
-
+ }
+
if (providerNames.contains(RtmrProviderName)) {
resultProviderDataSources.push_back(TString(RtmrProviderName));
}
@@ -1330,17 +1330,17 @@ TTypeAnnotationContextPtr TProgram::BuildTypeAnnotationContext(const TString& us
*FunctionRegistry_, IDataProvider::EResultFormat::Yson, ToString((ui32)resultFormat), writerFactory);
ResultProviderConfig_->SupportsResultPosition = SupportsResultPosition_;
auto resultProvider = CreateResultProvider(ResultProviderConfig_);
- typeAnnotationContext->AddDataSink(ResultProviderName, resultProvider);
+ typeAnnotationContext->AddDataSink(ResultProviderName, resultProvider);
typeAnnotationContext->AvailablePureResultDataSources = resultProviderDataSources;
}
- if (!fullResultDataSinks.empty()) {
- typeAnnotationContext->FullResultDataSink = fullResultDataSinks.front();
- }
-
+ if (!fullResultDataSinks.empty()) {
+ typeAnnotationContext->FullResultDataSink = fullResultDataSinks.front();
+ }
+
{
- auto configProvider = CreateConfigProvider(*typeAnnotationContext, GatewaysConfig_);
- typeAnnotationContext->AddDataSource(ConfigProviderName, configProvider);
+ auto configProvider = CreateConfigProvider(*typeAnnotationContext, GatewaysConfig_);
+ typeAnnotationContext->AddDataSource(ConfigProviderName, configProvider);
}
typeAnnotationContext->UserDataStorage->SetTokenResolver(BuildCompositeTokenResolver(std::move(tokenResolvers)));
@@ -1353,7 +1353,7 @@ TFuture<void> TProgram::OpenSession(const TString& username)
for (const auto& dp : DataProviders_) {
if (dp.OpenSession) {
auto future = dp.OpenSession(SessionId_, username, ProgressWriter_, OperationOptions_,
- RandomProvider_, TimeProvider_);
+ RandomProvider_, TimeProvider_);
openFutures.push_back(future);
}
}
@@ -1375,7 +1375,7 @@ void TProgram::Print(IOutputStream* exprOut, IOutputStream* planOut, bool cleanP
GetPlanBuilder().Clear();
}
printTransformers.push_back(TTransformStage(
- TPlanOutputTransformer::Sync(planOut, GetPlanBuilder(), OutputFormat_),
+ TPlanOutputTransformer::Sync(planOut, GetPlanBuilder(), OutputFormat_),
"PlanOutput",
issueCode));
}
diff --git a/ydb/library/yql/core/facade/yql_facade.h b/ydb/library/yql/core/facade/yql_facade.h
index 1f8f47681d..a19b74b562 100644
--- a/ydb/library/yql/core/facade/yql_facade.h
+++ b/ydb/library/yql/core/facade/yql_facade.h
@@ -7,13 +7,13 @@
#include <ydb/library/yql/providers/result/provider/yql_result_provider.h>
#include <ydb/library/yql/core/yql_type_annotation.h>
#include <ydb/library/yql/sql/sql.h>
-
+
#include <library/cpp/random_provider/random_provider.h>
#include <library/cpp/time_provider/time_provider.h>
#include <library/cpp/threading/future/future.h>
#include <util/system/file.h>
-#include <util/generic/ptr.h>
+#include <util/generic/ptr.h>
#include <functional>
@@ -25,16 +25,16 @@ namespace NMiniKQL {
namespace NYql {
-class TGatewaysConfig;
+class TGatewaysConfig;
class TProgram;
-using TProgramPtr = TIntrusivePtr<TProgram>;
-class TProgramFactory;
-using TProgramFactoryPtr = TIntrusivePtr<TProgramFactory>;
+using TProgramPtr = TIntrusivePtr<TProgram>;
+class TProgramFactory;
+using TProgramFactoryPtr = TIntrusivePtr<TProgramFactory>;
///////////////////////////////////////////////////////////////////////////////
// TProgramFactory
///////////////////////////////////////////////////////////////////////////////
-class TProgramFactory: public TThrRefBase, private TMoveOnly
+class TProgramFactory: public TThrRefBase, private TMoveOnly
{
public:
TProgramFactory(
@@ -42,14 +42,14 @@ public:
const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
ui64 nextUniqueId,
const TVector<TDataProviderInitializer>& dataProvidersInit,
- const TString& runner);
+ const TString& runner);
void AddUserDataTable(const TUserDataTable& userDataTable);
void AddCredentialsTable(TCredentialTablePtr credentialTable);
void SetUserCredentials(const TUserCredentials& userCredentials);
void SetGatewaysConfig(const TGatewaysConfig* gatewaysConfig);
void SetModules(IModuleResolver::TPtr modules);
- void SetUdfResolver(IUdfResolver::TPtr udfResolver);
+ void SetUdfResolver(IUdfResolver::TPtr udfResolver);
void SetUdfIndex(TUdfIndex::TPtr udfIndex, TUdfIndexPackageSet::TPtr udfIndexPackageSet);
void SetFileStorage(TFileStoragePtr fileStorage);
void EnableRangeComputeFor();
@@ -75,18 +75,18 @@ private:
TUserCredentials UserCredentials_;
const TGatewaysConfig* GatewaysConfig_;
IModuleResolver::TPtr Modules_;
- IUdfResolver::TPtr UdfResolver_;
+ IUdfResolver::TPtr UdfResolver_;
TUdfIndex::TPtr UdfIndex_;
TUdfIndexPackageSet::TPtr UdfIndexPackageSet_;
TFileStoragePtr FileStorage_;
- TString Runner_;
+ TString Runner_;
bool EnableRangeComputeFor_ = false;
};
///////////////////////////////////////////////////////////////////////////////
// TProgram
///////////////////////////////////////////////////////////////////////////////
-class TProgram: public TThrRefBase, private TNonCopyable
+class TProgram: public TThrRefBase, private TNonCopyable
{
public:
friend TProgramFactory;
@@ -102,10 +102,10 @@ public:
bool Compile(const TString& username);
- TStatus Discover(const TString& username);
-
- TFutureStatus DiscoverAsync(const TString& username);
-
+ TStatus Discover(const TString& username);
+
+ TFutureStatus DiscoverAsync(const TString& username);
+
TStatus Validate(const TString& username, IOutputStream* exprOut = nullptr, bool withTypes = false);
TFutureStatus ValidateAsync(const TString& username, IOutputStream* exprOut = nullptr, bool withTypes = false);
@@ -160,19 +160,19 @@ public:
void Abort();
- inline TIssues Issues() {
+ inline TIssues Issues() {
if (ExprCtx_) {
return ExprCtx_->IssueManager.GetIssues();
} else {
- return {};
+ return {};
}
}
- inline TIssues CompletedIssues() const {
+ inline TIssues CompletedIssues() const {
if (ExprCtx_) {
return ExprCtx_->IssueManager.GetCompletedIssues();
} else {
- return {};
+ return {};
}
}
@@ -213,14 +213,14 @@ public:
}
TMaybe<TString> GetDiagnostics();
- IGraphTransformer::TStatistics GetRawDiagnostics();
+ IGraphTransformer::TStatistics GetRawDiagnostics();
TMaybe<TString> GetTasksInfo();
- TMaybe<TString> GetStatistics(bool totalOnly = false);
-
- TMaybe<TString> GetDiscoveredData();
+ TMaybe<TString> GetStatistics(bool totalOnly = false);
+ TMaybe<TString> GetDiscoveredData();
+
TString ResultsAsString() const;
void ConfigureYsonResultFormat(NYson::EYsonFormat format);
@@ -232,7 +232,7 @@ public:
void SetValidateOptions(NUdf::EValidateMode validateMode);
void SetDisableNativeUdfSupport(bool disable);
- void SetUseTableMetaFromGraph(bool use);
+ void SetUseTableMetaFromGraph(bool use);
void SetProgressWriter(TOperationProgressWriter writer) {
Y_ENSURE(!TypeCtx_, "TypeCtx_ already created");
@@ -263,16 +263,16 @@ public:
OperationOptions_.Title = title;
}
- void SetOperationUrl(const TString& url) {
+ void SetOperationUrl(const TString& url) {
+ Y_ENSURE(!TypeCtx_, "TypeCtx_ already created");
+ OperationOptions_.Url = url;
+ }
+
+ void SetQueryName(const TString& name) {
Y_ENSURE(!TypeCtx_, "TypeCtx_ already created");
- OperationOptions_.Url = url;
- }
-
- void SetQueryName(const TString& name) {
- Y_ENSURE(!TypeCtx_, "TypeCtx_ already created");
- OperationOptions_.QueryName = name;
- }
-
+ OperationOptions_.QueryName = name;
+ }
+
void SetOperationAttrsYson(const TString& attrs) {
Y_ENSURE(!TypeCtx_, "TypeCtx_ already created");
OperationOptions_.AttrsYson = attrs;
@@ -303,14 +303,14 @@ private:
const TVector<TCredentialTablePtr>& credentialTables,
const TUserCredentials& userCredentials,
const IModuleResolver::TPtr& modules,
- const IUdfResolver::TPtr& udfResolver,
+ const IUdfResolver::TPtr& udfResolver,
const TUdfIndex::TPtr& udfIndex,
const TUdfIndexPackageSet::TPtr& udfIndexPackageSet,
const TFileStoragePtr& fileStorage,
const TGatewaysConfig* gatewaysConfig,
const TString& filename,
const TString& sourceCode,
- const TString& sessionId,
+ const TString& sessionId,
const TString& runner,
bool enableRangeComputeFor);
@@ -321,7 +321,7 @@ private:
NThreading::TFuture<void> OpenSession(const TString& username);
- void CleanupLastSession();
+ void CleanupLastSession();
void CloseLastSession();
TFutureStatus RemoteKikimrValidate(const TString& cluster);
@@ -344,7 +344,7 @@ private:
TYqlOperationOptions OperationOptions_;
TVector<TCredentialTablePtr> CredentialTables_;
TUserCredentials UserCredentials_;
- const IUdfResolver::TPtr UdfResolver_;
+ const IUdfResolver::TPtr UdfResolver_;
const TUdfIndex::TPtr UdfIndex_;
const TUdfIndexPackageSet::TPtr UdfIndexPackageSet_;
const TFileStoragePtr FileStorage_;
@@ -373,7 +373,7 @@ private:
TMaybe<NYson::EYsonFormat> DiagnosticFormat_;
NUdf::EValidateMode ValidateMode_ = NUdf::EValidateMode::None;
bool DisableNativeUdfSupport_ = false;
- bool UseTableMetaFromGraph_ = false;
+ bool UseTableMetaFromGraph_ = false;
TMaybe<TSet<TString>> UsedClusters_;
TMaybe<TSet<TString>> UsedProviders_;
TMaybe<TString> ExternalQueryAst_;
diff --git a/ydb/library/yql/core/file_storage/file_storage.cpp b/ydb/library/yql/core/file_storage/file_storage.cpp
index 631adbc688..84f9977b7e 100644
--- a/ydb/library/yql/core/file_storage/file_storage.cpp
+++ b/ydb/library/yql/core/file_storage/file_storage.cpp
@@ -1,91 +1,91 @@
-#include "file_storage.h"
-#include "storage.h"
+#include "file_storage.h"
+#include "storage.h"
#include "url_mapper.h"
#include "url_meta.h"
-#include "download_stream.h"
-
-#include <ydb/library/yql/core/file_storage/proto/file_storage.pb.h>
-
+#include "download_stream.h"
+
+#include <ydb/library/yql/core/file_storage/proto/file_storage.pb.h>
+
#include <ydb/library/yql/utils/fetch/fetch.h>
#include <ydb/library/yql/utils/log/log.h>
-#include <ydb/library/yql/utils/log/context.h>
+#include <ydb/library/yql/utils/log/context.h>
#include <ydb/library/yql/utils/multi_resource_lock.h>
#include <ydb/library/yql/utils/md5_stream.h>
#include <ydb/library/yql/utils/retry.h>
-#include <ydb/library/yql/utils/yql_panic.h>
+#include <ydb/library/yql/utils/yql_panic.h>
#include <library/cpp/cache/cache.h>
#include <library/cpp/digest/md5/md5.h>
#include <library/cpp/threading/future/async.h>
-
+
#include <util/generic/guid.h>
-#include <util/generic/yexception.h>
-
+#include <util/generic/yexception.h>
+
#include <util/stream/file.h>
#include <util/stream/null.h>
-#include <util/system/fs.h>
+#include <util/system/fs.h>
#include <util/system/fstat.h>
#include <util/system/guard.h>
#include <util/system/shellcommand.h>
-#include <util/system/sysstat.h>
+#include <util/system/sysstat.h>
#include <util/system/utime.h>
-
-namespace NYql {
+
+namespace NYql {
class TFileStorageImpl: public IFileStorage {
-public:
+public:
explicit TFileStorageImpl(const TFileStorageConfig& params)
: Storage(params.GetMaxFiles(), ui64(params.GetMaxSizeMb()) << 20ull, params.GetPath())
, Config(params)
, QueueStarted(0)
- {
- try {
+ {
+ try {
for (const auto& sc : params.GetCustomSchemes()) {
Mapper.AddMapping(sc.GetPattern(), sc.GetTargetUrl());
- }
- } catch (const yexception& e) {
+ }
+ } catch (const yexception& e) {
ythrow yexception() << "FileStorage: " << e.what();
- }
+ }
auto numThreads = params.GetThreads();
if (1 == numThreads) {
MtpQueue.Reset(new TFakeThreadPool());
} else {
- MtpQueue.Reset(new TSimpleThreadPool(TThreadPoolParams{"FileStorage"}));
+ MtpQueue.Reset(new TSimpleThreadPool(TThreadPoolParams{"FileStorage"}));
}
// do not call MtpQueue->Start here as we have to do it _after_ fork
- }
-
+ }
+
~TFileStorageImpl() {
MtpQueue->Stop();
}
- void AddDownloader(IDownloaderPtr downloader) override {
- Downloaders.push_back(std::move(downloader));
+ void AddDownloader(IDownloaderPtr downloader) override {
+ Downloaders.push_back(std::move(downloader));
}
TFileLinkPtr PutFile(const TString& file, const TString& outFileName = {}) override {
- YQL_LOG(INFO) << "PutFile to cache: " << file;
+ YQL_LOG(INFO) << "PutFile to cache: " << file;
const auto md5 = MD5::File(file);
const TString storageFileName = md5 + ".file";
auto lock = MultiResourceLock.Acquire(storageFileName);
return Storage.Put(storageFileName, outFileName, md5, [&file, &md5](const TFsPath& dstFile) {
- NFs::HardLinkOrCopy(file, dstFile);
- i64 length = GetFileLength(dstFile.c_str());
- if (length == -1) {
- ythrow TSystemError() << "cannot get file length: " << dstFile;
- }
- i64 srcLength = GetFileLength(file.c_str());
- if (srcLength == -1) {
- ythrow TSystemError() << "cannot get file length: " << file;
- }
-
- YQL_ENSURE(srcLength == length);
- return std::make_pair(static_cast<ui64>(length), md5);
- });
- }
-
+ NFs::HardLinkOrCopy(file, dstFile);
+ i64 length = GetFileLength(dstFile.c_str());
+ if (length == -1) {
+ ythrow TSystemError() << "cannot get file length: " << dstFile;
+ }
+ i64 srcLength = GetFileLength(file.c_str());
+ if (srcLength == -1) {
+ ythrow TSystemError() << "cannot get file length: " << file;
+ }
+
+ YQL_ENSURE(srcLength == length);
+ return std::make_pair(static_cast<ui64>(length), md5);
+ });
+ }
+
TFileLinkPtr PutFileStripped(const TString& file, const TString& originalMd5 = {}) override {
YQL_LOG(INFO) << "PutFileStripped to cache: " << file;
if (originalMd5.empty()) {
@@ -106,17 +106,17 @@ public:
strippedMeta = TUrlMeta();
const TString storageFileName = md5 + ".file.stripped";
TFileLinkPtr result = Storage.Put(storageFileName, "", "", [&file](const TFsPath& dstPath) {
- ui64 size;
- TString md5;
- TShellCommand cmd("strip", {file, "-o", dstPath.GetPath()});
- cmd.Run().Wait();
- if (*cmd.GetExitCode() != 0) {
- ythrow yexception() << cmd.GetError();
+ ui64 size;
+ TString md5;
+ TShellCommand cmd("strip", {file, "-o", dstPath.GetPath()});
+ cmd.Run().Wait();
+ if (*cmd.GetExitCode() != 0) {
+ ythrow yexception() << cmd.GetError();
}
- md5 = MD5::File(dstPath.GetPath());
- size = TFile(dstPath.GetPath(), OpenExisting | RdOnly).GetLength();
- YQL_LOG(DEBUG) << "Strip " << file << " to " << dstPath.GetPath();
- return std::make_pair(size, md5);
+ md5 = MD5::File(dstPath.GetPath());
+ size = TFile(dstPath.GetPath(), OpenExisting | RdOnly).GetLength();
+ YQL_LOG(DEBUG) << "Strip " << file << " to " << dstPath.GetPath();
+ return std::make_pair(size, md5);
});
strippedMeta.ContentFile = result->GetStorageFileName();
@@ -130,47 +130,47 @@ public:
TFileLinkPtr PutInline(const TString& data) override {
const auto md5 = MD5::Calc(data);
const TString storageFileName = md5 + ".file";
- YQL_LOG(INFO) << "PutInline to cache. md5=" << md5;
+ YQL_LOG(INFO) << "PutInline to cache. md5=" << md5;
auto lock = MultiResourceLock.Acquire(storageFileName);
return Storage.Put(storageFileName, TString(), md5, [&data, &md5](const TFsPath& dstFile) {
- TStringInput in(data);
- TFile outFile(dstFile, CreateAlways | ARW | AX);
- TUnbufferedFileOutput out(outFile);
- auto result = std::make_pair(TransferData(&in, &out), md5);
- outFile.Close();
-
- i64 length = GetFileLength(dstFile.c_str());
- if (length == -1) {
- ythrow TSystemError() << "cannot get file length: " << dstFile;
- }
-
- YQL_ENSURE(data.size() == static_cast<ui64>(length));
- return result;
- });
- }
-
+ TStringInput in(data);
+ TFile outFile(dstFile, CreateAlways | ARW | AX);
+ TUnbufferedFileOutput out(outFile);
+ auto result = std::make_pair(TransferData(&in, &out), md5);
+ outFile.Close();
+
+ i64 length = GetFileLength(dstFile.c_str());
+ if (length == -1) {
+ ythrow TSystemError() << "cannot get file length: " << dstFile;
+ }
+
+ YQL_ENSURE(data.size() == static_cast<ui64>(length));
+ return result;
+ });
+ }
+
TFileLinkPtr PutUrl(const TString& urlStr, const TString& oauthToken) override {
try {
TString convertedUrl;
if (!Mapper.MapUrl(urlStr, convertedUrl)) {
convertedUrl = urlStr;
}
-
- YQL_LOG(INFO) << "PutUrl to cache: " << convertedUrl;
+
+ YQL_LOG(INFO) << "PutUrl to cache: " << convertedUrl;
THttpURL url = ParseURL(convertedUrl);
- for (const auto& d: Downloaders) {
- if (d->Accept(url)) {
- return PutUrl(url, oauthToken, d);
+ for (const auto& d: Downloaders) {
+ if (d->Accept(url)) {
+ return PutUrl(url, oauthToken, d);
}
}
-
- ythrow yexception() << "Unsupported url: " << convertedUrl;
+
+ ythrow yexception() << "Unsupported url: " << convertedUrl;
} catch (const std::exception& e) {
- YQL_LOG(ERROR) << "Failed to download file by URL \"" << urlStr << "\", details: " << e.what();
+ YQL_LOG(ERROR) << "Failed to download file by URL \"" << urlStr << "\", details: " << e.what();
YQL_LOG_CTX_THROW yexception() << "FileStorage: Failed to download file by URL \"" << urlStr << "\", details: " << e.what();
- }
- }
-
+ }
+ }
+
NThreading::TFuture<TFileLinkPtr> PutFileAsync(const TString& file, const TString& outFileName = {}) override {
StartQueueOnce();
return NThreading::Async([=]() {
@@ -194,17 +194,17 @@ public:
TFsPath GetRoot() const override {
return Storage.GetRoot();
- }
-
+ }
+
TFsPath GetTemp() const override {
return Storage.GetTemp();
- }
-
- const TFileStorageConfig& GetConfig() const override {
- return Config;
- }
-
-private:
+ }
+
+ const TFileStorageConfig& GetConfig() const override {
+ return Config;
+ }
+
+private:
void StartQueueOnce() {
// we shall call Start only once
if (AtomicTryLock(&QueueStarted)) {
@@ -212,16 +212,16 @@ private:
}
}
- TFileLinkPtr PutUrl(const THttpURL& url, const TString& oauthToken, const IDownloaderPtr& downloader) {
+ TFileLinkPtr PutUrl(const THttpURL& url, const TString& oauthToken, const IDownloaderPtr& downloader) {
return WithRetry<TDownloadError>(Config.GetRetryCount(), [&, this]() {
- return this->DoPutUrl(url, oauthToken, downloader);
+ return this->DoPutUrl(url, oauthToken, downloader);
}, [&](const auto& e, int attempt, int attemptCount) {
- YQL_LOG(WARN) << "Error while downloading url " << url.PrintS() << ", attempt " << attempt << "/" << attemptCount << ", details: " << e.what();
+ YQL_LOG(WARN) << "Error while downloading url " << url.PrintS() << ", attempt " << attempt << "/" << attemptCount << ", details: " << e.what();
Sleep(TDuration::MilliSeconds(Config.GetRetryDelayMs()));
});
}
- TFileLinkPtr DoPutUrl(const THttpURL& url, const TString& oauthToken, const IDownloaderPtr& downloader) {
+ TFileLinkPtr DoPutUrl(const THttpURL& url, const TString& oauthToken, const IDownloaderPtr& downloader) {
const auto urlMetaFile = BuildUrlMetaFileName(url);
auto lock = MultiResourceLock.Acquire(urlMetaFile); // let's use meta file as lock name
@@ -237,35 +237,35 @@ private:
urlMeta = TUrlMeta();
}
- YQL_LOG(INFO) << "UrlMeta: " << urlMetaFile << ", ETag=" << urlMeta.ETag
- << ", ContentFile=" << urlMeta.ContentFile << ", Md5=" << urlMeta.Md5
- << ", LastModified=" << urlMeta.LastModified;
-
- TStorage::TDataPuller puller;
+ YQL_LOG(INFO) << "UrlMeta: " << urlMetaFile << ", ETag=" << urlMeta.ETag
+ << ", ContentFile=" << urlMeta.ContentFile << ", Md5=" << urlMeta.Md5
+ << ", LastModified=" << urlMeta.LastModified;
+
+ TStorage::TDataPuller puller;
TString etag;
TString lastModified;
- std::tie(puller, etag, lastModified) = downloader->Download(url, oauthToken, urlMeta.ETag, urlMeta.LastModified);
- if (!puller) {
- Y_ENSURE(oldContentLink); // should not fire
- return oldContentLink;
- }
- if (urlMeta.ETag && urlMeta.ETag == etag && oldContentLink) {
- // for non-empty etags server should reply with code 304 Not modified
- // but some servers like github.com do not support IfNoneMatch (but ETag is not empty)
- // no etag is supported (previously and now) or server responded with status code 200 and we have something in cache
- // access already checked by FetchWithETagAndLastModified
- return oldContentLink;
- }
- if (urlMeta.ETag && etag) {
- YQL_LOG(INFO) << "ETag for url " << url.PrintS() << " has been changed from " << urlMeta.ETag << " to " << etag << ". We have to download new version";
- }
- else if (urlMeta.LastModified && lastModified) {
- YQL_LOG(INFO) << "LastModified for url " << url.PrintS() << " has been changed from " << urlMeta.LastModified << " to " << lastModified << ". We have to download new version";
- }
-
- // todo: remove oldContentLink ?
- const auto urlContentFile = BuildUrlContentFileName(url, etag, lastModified);
- TFileLinkPtr result = Storage.Put(urlContentFile, TString(), TString(), puller);
+ std::tie(puller, etag, lastModified) = downloader->Download(url, oauthToken, urlMeta.ETag, urlMeta.LastModified);
+ if (!puller) {
+ Y_ENSURE(oldContentLink); // should not fire
+ return oldContentLink;
+ }
+ if (urlMeta.ETag && urlMeta.ETag == etag && oldContentLink) {
+ // for non-empty etags server should reply with code 304 Not modified
+ // but some servers like github.com do not support IfNoneMatch (but ETag is not empty)
+ // no etag is supported (previously and now) or server responded with status code 200 and we have something in cache
+ // access already checked by FetchWithETagAndLastModified
+ return oldContentLink;
+ }
+ if (urlMeta.ETag && etag) {
+ YQL_LOG(INFO) << "ETag for url " << url.PrintS() << " has been changed from " << urlMeta.ETag << " to " << etag << ". We have to download new version";
+ }
+ else if (urlMeta.LastModified && lastModified) {
+ YQL_LOG(INFO) << "LastModified for url " << url.PrintS() << " has been changed from " << urlMeta.LastModified << " to " << lastModified << ". We have to download new version";
+ }
+
+ // todo: remove oldContentLink ?
+ const auto urlContentFile = BuildUrlContentFileName(url, etag, lastModified);
+ TFileLinkPtr result = Storage.Put(urlContentFile, TString(), TString(), puller);
// save meta using rename for atomicity
urlMeta.ETag = etag;
@@ -278,8 +278,8 @@ private:
Storage.MoveToStorage(metaTmpFile, urlMetaFile);
return result;
- }
-
+ }
+
static TString BuildUrlMetaFileName(const THttpURL& url) {
return MD5::Calc(url.PrintS(THttpURL::FlagNoFrag | THttpURL::FlagHostAscii)) + ".url_meta";
}
@@ -289,20 +289,20 @@ private:
return MD5::Calc(needle + url.PrintS(THttpURL::FlagNoFrag | THttpURL::FlagHostAscii)) + ".url";
}
-private:
+private:
TStorage Storage;
const TFileStorageConfig Config;
- std::vector<IDownloaderPtr> Downloaders;
+ std::vector<IDownloaderPtr> Downloaders;
TUrlMapper Mapper;
TAtomic QueueStarted;
THolder<IThreadPool> MtpQueue;
TMultiResourceLock MultiResourceLock;
-};
-
+};
+
TFileStoragePtr CreateFileStorage(const TFileStorageConfig& params) {
- Y_ENSURE(0 != params.GetMaxFiles(), "FileStorage: MaxFiles must be greater than 0");
- Y_ENSURE(0 != params.GetMaxSizeMb(), "FileStorage: MaxSizeMb must be greater than 0");
- return new TFileStorageImpl(params);
-}
-
-} // NYql
+ Y_ENSURE(0 != params.GetMaxFiles(), "FileStorage: MaxFiles must be greater than 0");
+ Y_ENSURE(0 != params.GetMaxSizeMb(), "FileStorage: MaxSizeMb must be greater than 0");
+ return new TFileStorageImpl(params);
+}
+
+} // NYql
diff --git a/ydb/library/yql/core/file_storage/file_storage.h b/ydb/library/yql/core/file_storage/file_storage.h
index 9133d57b1b..0da0c3541c 100644
--- a/ydb/library/yql/core/file_storage/file_storage.h
+++ b/ydb/library/yql/core/file_storage/file_storage.h
@@ -1,32 +1,32 @@
-#pragma once
-
+#pragma once
+
#include "storage.h"
#include <library/cpp/threading/future/future.h>
-#include <library/cpp/uri/http_url.h>
+#include <library/cpp/uri/http_url.h>
-#include <util/folder/path.h>
-#include <util/generic/ptr.h>
+#include <util/folder/path.h>
+#include <util/generic/ptr.h>
#include <util/generic/string.h>
-
-#include <functional>
-#include <unordered_map>
-#include <tuple>
-
-namespace NYql {
-
-class TFileStorageConfig;
-
-struct IFileStorage: public TThrRefBase {
- struct IDownloader : public TThrRefBase {
- virtual bool Accept(const THttpURL& url) = 0;
- virtual std::tuple<TStorage::TDataPuller, TString, TString> Download(const THttpURL& url, const TString& oauthToken, const TString& etag, const TString& lastModified) = 0;
- };
- using IDownloaderPtr = TIntrusivePtr<IDownloader>;
-
+
+#include <functional>
+#include <unordered_map>
+#include <tuple>
+
+namespace NYql {
+
+class TFileStorageConfig;
+
+struct IFileStorage: public TThrRefBase {
+ struct IDownloader : public TThrRefBase {
+ virtual bool Accept(const THttpURL& url) = 0;
+ virtual std::tuple<TStorage::TDataPuller, TString, TString> Download(const THttpURL& url, const TString& oauthToken, const TString& etag, const TString& lastModified) = 0;
+ };
+ using IDownloaderPtr = TIntrusivePtr<IDownloader>;
+
virtual ~IFileStorage() = default;
- virtual void AddDownloader(IDownloaderPtr downloader) = 0;
- virtual TFileLinkPtr PutFile(const TString& file, const TString& outFileName = {}) = 0;
+ virtual void AddDownloader(IDownloaderPtr downloader) = 0;
+ virtual TFileLinkPtr PutFile(const TString& file, const TString& outFileName = {}) = 0;
virtual TFileLinkPtr PutFileStripped(const TString& file, const TString& originalMd5 = {}) = 0;
virtual TFileLinkPtr PutInline(const TString& data) = 0;
virtual TFileLinkPtr PutUrl(const TString& url, const TString& oauthToken) = 0;
@@ -35,14 +35,14 @@ struct IFileStorage: public TThrRefBase {
virtual NThreading::TFuture<TFileLinkPtr> PutInlineAsync(const TString& data) = 0;
virtual NThreading::TFuture<TFileLinkPtr> PutUrlAsync(const TString& url, const TString& oauthToken) = 0;
- virtual TFsPath GetRoot() const = 0;
- virtual TFsPath GetTemp() const = 0;
- virtual const TFileStorageConfig& GetConfig() const = 0;
-};
-
-using TFileStoragePtr = TIntrusivePtr<IFileStorage>;
-
-// Will use auto-cleaned temporary directory if storagePath is empty
+ virtual TFsPath GetRoot() const = 0;
+ virtual TFsPath GetTemp() const = 0;
+ virtual const TFileStorageConfig& GetConfig() const = 0;
+};
+
+using TFileStoragePtr = TIntrusivePtr<IFileStorage>;
+
+// Will use auto-cleaned temporary directory if storagePath is empty
TFileStoragePtr CreateFileStorage(const TFileStorageConfig& params);
-
-} // NYql
+
+} // NYql
diff --git a/ydb/library/yql/core/file_storage/file_storage_ut.cpp b/ydb/library/yql/core/file_storage/file_storage_ut.cpp
index 5103fdd31d..539f7af6cc 100644
--- a/ydb/library/yql/core/file_storage/file_storage_ut.cpp
+++ b/ydb/library/yql/core/file_storage/file_storage_ut.cpp
@@ -1,8 +1,8 @@
#include "file_storage.h"
-
+
#include <ydb/library/yql/core/file_storage/ut/test_http_server.h>
#include <ydb/library/yql/core/file_storage/proto/file_storage.pb.h>
-#include <ydb/library/yql/core/file_storage/http_download/http_download.h>
+#include <ydb/library/yql/core/file_storage/http_download/http_download.h>
#include <library/cpp/threading/future/future.h>
#include <library/cpp/threading/future/async.h>
@@ -22,12 +22,12 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TIFStream(path).ReadAll();
}
- static TFileStoragePtr CreateTestFS(const TFileStorageConfig& params = {}, const std::vector<TString>& extraPatterns = {}) {
- TFileStoragePtr fs = CreateFileStorage(params);
- fs->AddDownloader(MakeHttpDownloader(false, params, extraPatterns));
- return fs;
- }
-
+ static TFileStoragePtr CreateTestFS(const TFileStorageConfig& params = {}, const std::vector<TString>& extraPatterns = {}) {
+ TFileStoragePtr fs = CreateFileStorage(params);
+ fs->AddDownloader(MakeHttpDownloader(false, params, extraPatterns));
+ return fs;
+ }
+
static std::unique_ptr<TTestHttpServer> CreateTestHttpServer() {
TPortManager pm;
const ui16 port = pm.GetPort();
@@ -64,7 +64,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TTestHttpServer::TReply::Ok(currentContent);
});
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
auto url = server->GetUrl();
auto link1 = fs->PutUrl(url, {});
@@ -97,7 +97,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TTestHttpServer::TReply::Ok(content);
});
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
auto url = server->GetUrl();
@@ -126,7 +126,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TTestHttpServer::TReply::OkETag(currentContent, currentETag);
});
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
auto url = server->GetUrl();
@@ -169,7 +169,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TTestHttpServer::TReply::OkLastModified(currentContent, currentLastModified);
});
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
auto url = server->GetUrl();
@@ -209,7 +209,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TTestHttpServer::TReply::OkETag(currentContent, currentETag);
});
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
auto url = server->GetUrl();
@@ -252,7 +252,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TTestHttpServer::TReply::OkETag(currentContent, MakeWeakETag(currentETag));
});
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
auto url = server->GetUrl();
@@ -288,7 +288,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TTestHttpServer::TReply::Ok(currentContent);
});
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
auto url = server->GetUrl();
auto link1 = fs->PutUrl(url, {});
@@ -318,7 +318,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TTestHttpServer::TReply::OkETag(currentContent, currentETag);
});
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
auto url = server->GetUrl();
auto link1 = fs->PutUrl(url, {});
@@ -348,7 +348,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TTestHttpServer::TReply::OkETag(currentContent, currentETag);
});
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
auto url = server->GetUrl();
auto link1 = fs->PutUrl(url, {});
@@ -369,7 +369,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
Y_UNIT_TEST(Md5ForPutFiles) {
TString currentContent = "ABC";
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
TTempFileHandle h1;
h1.Write("ABC", 3);
@@ -409,7 +409,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
TFileStorageConfig params;
params.SetRetryCount(3);
- TFileStoragePtr fs = CreateTestFS(params);
+ TFileStoragePtr fs = CreateTestFS(params);
auto url = server->GetUrl();
@@ -429,7 +429,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
return TTestHttpServer::TReply::OkETag(currentContent, currentETag, 0);
});
- TFileStoragePtr fs = CreateTestFS();
+ TFileStoragePtr fs = CreateTestFS();
auto url = server->GetUrl();
@@ -458,7 +458,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
TFileStorageConfig params;
params.SetRetryCount(3);
- TFileStoragePtr fs = CreateTestFS(params);
+ TFileStoragePtr fs = CreateTestFS(params);
auto url = server->GetUrl();
@@ -476,36 +476,36 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
auto url = server->GetUrl();
- {
- // not in whitelist
- TFileStorageConfig params;
- params.AddAllowedUrlPatterns("^XXXX$");
- TFileStoragePtr fs = CreateTestFS(params);
-
- UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, {}), std::exception, "It is not allowed to download url http://localhost:");
- }
-
- {
- // have in whitelist
- TFileStorageConfig params;
- params.SetSocketTimeoutMs(4000);
- params.AddAllowedUrlPatterns("^http://localhost:");
- TFileStoragePtr fs = CreateTestFS(params);
-
- auto link = fs->PutUrl(url, {});
- UNIT_ASSERT_VALUES_EQUAL(currentContent, ReadFileContent(link->GetPath()));
- }
-
- {
- // have eaxtra url in whitelist
- TFileStorageConfig params;
- params.SetSocketTimeoutMs(4000);
- params.AddAllowedUrlPatterns("^XXXX$");
- TFileStoragePtr fs = CreateTestFS(params, std::vector{TString{"^http://localhost:"}});
-
- auto link = fs->PutUrl(url, {});
- UNIT_ASSERT_VALUES_EQUAL(currentContent, ReadFileContent(link->GetPath()));
- }
+ {
+ // not in whitelist
+ TFileStorageConfig params;
+ params.AddAllowedUrlPatterns("^XXXX$");
+ TFileStoragePtr fs = CreateTestFS(params);
+
+ UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, {}), std::exception, "It is not allowed to download url http://localhost:");
+ }
+
+ {
+ // have in whitelist
+ TFileStorageConfig params;
+ params.SetSocketTimeoutMs(4000);
+ params.AddAllowedUrlPatterns("^http://localhost:");
+ TFileStoragePtr fs = CreateTestFS(params);
+
+ auto link = fs->PutUrl(url, {});
+ UNIT_ASSERT_VALUES_EQUAL(currentContent, ReadFileContent(link->GetPath()));
+ }
+
+ {
+ // have eaxtra url in whitelist
+ TFileStorageConfig params;
+ params.SetSocketTimeoutMs(4000);
+ params.AddAllowedUrlPatterns("^XXXX$");
+ TFileStoragePtr fs = CreateTestFS(params, std::vector{TString{"^http://localhost:"}});
+
+ auto link = fs->PutUrl(url, {});
+ UNIT_ASSERT_VALUES_EQUAL(currentContent, ReadFileContent(link->GetPath()));
+ }
}
Y_UNIT_TEST(SocketTimeout) {
@@ -518,7 +518,7 @@ Y_UNIT_TEST_SUITE(TFileStorageTests) {
TFileStorageConfig params;
params.SetSocketTimeoutMs(1000);
- TFileStoragePtr fs = CreateTestFS(params);
+ TFileStoragePtr fs = CreateTestFS(params);
auto url = server->GetUrl();
UNIT_ASSERT_EXCEPTION_CONTAINS(fs->PutUrl(url, {}), std::exception, "can not read from socket input stream");
diff --git a/ydb/library/yql/core/file_storage/http_download/http_download.cpp b/ydb/library/yql/core/file_storage/http_download/http_download.cpp
index 0011d3b92a..6d5ef34231 100644
--- a/ydb/library/yql/core/file_storage/http_download/http_download.cpp
+++ b/ydb/library/yql/core/file_storage/http_download/http_download.cpp
@@ -1,172 +1,172 @@
-#include "http_download.h"
-#include "pattern_group.h"
-
-#include <ydb/library/yql/core/file_storage/proto/file_storage.pb.h>
-#include <ydb/library/yql/core/file_storage/download_stream.h>
-
-#include <ydb/library/yql/utils/fetch/fetch.h>
-#include <ydb/library/yql/utils/log/log.h>
-#include <ydb/library/yql/utils/log/context.h>
-#include <ydb/library/yql/utils/multi_resource_lock.h>
-#include <ydb/library/yql/utils/md5_stream.h>
-#include <ydb/library/yql/utils/retry.h>
-#include <ydb/library/yql/utils/yql_panic.h>
-
-#include <library/cpp/digest/md5/md5.h>
-#include <library/cpp/http/misc/httpcodes.h>
-
-#include <util/generic/guid.h>
-#include <util/generic/yexception.h>
-#include <util/stream/file.h>
-#include <util/system/file.h>
-
-
-namespace NYql {
-
-class THttpDownloader: public IFileStorage::IDownloader {
-public:
- THttpDownloader(bool restictedUser, const TFileStorageConfig& config, const std::vector<TString>& extraAllowedUrls)
- : SocketTimeoutMs(config.GetSocketTimeoutMs())
- {
- for (const auto& p : restictedUser ? config.GetExternalAllowedUrlPatterns() : config.GetAllowedUrlPatterns()) {
- AllowedUrls.Add(p);
- }
- for (auto p: extraAllowedUrls) {
- AllowedUrls.Add(p);
- }
- }
- ~THttpDownloader() = default;
-
- bool Accept(const THttpURL& url) final {
- switch (url.GetScheme()) {
- case NUri::TScheme::SchemeHTTP:
- case NUri::TScheme::SchemeHTTPS:
- return true;
- default:
- break;
- }
- return false;
- }
-
- std::tuple<TStorage::TDataPuller, TString, TString> Download(const THttpURL& url, const TString& oauthToken, const TString& oldEtag, const TString& oldLastModified) final {
- if (!AllowedUrls.IsEmpty()) {
- if (auto strUrl = url.PrintS(); !AllowedUrls.Match(strUrl)) {
- YQL_LOG(WARN) << "FileStorage: url " << strUrl << " is not in whitelist, reject downloading";
- throw yexception() << "It is not allowed to download url " << strUrl;
- }
- }
-
- TFetchResultPtr fr1 = FetchWithETagAndLastModified(url, oauthToken, oldEtag, oldLastModified, SocketTimeoutMs);
- switch (fr1->GetRetCode()) {
- case HTTP_NOT_MODIFIED:
- return std::make_tuple(TStorage::TDataPuller{}, TString{}, TString{});
- case HTTP_OK:
- break;
- default:
- ythrow yexception() << "Url " << url.PrintS() << " cannot be accessed, code: " << fr1->GetRetCode();
- }
-
- auto pair = ExtractETagAndLastModified(*fr1);
-
- auto puller = [urlStr = url.PrintS(), fr1](const TFsPath& dstPath) -> std::pair<ui64, TString> {
- return CopyToFile(urlStr, *fr1, dstPath);
- };
-
- return std::make_tuple(puller, pair.first, pair.second);
- }
-
-private:
- static TFetchResultPtr FetchWithETagAndLastModified(const THttpURL& url, const TString& oauthToken, const TString& oldEtag, const TString& oldLastModified, ui32 socketTimeoutMs) {
- // more details about ETag and ModifiedSince: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.26
- THttpHeaders headers;
- if (!oauthToken.empty()) {
- headers.AddHeader(THttpInputHeader("Authorization", "OAuth " + oauthToken));
- }
-
- // ETag has priority over modification time
- if (!oldEtag.empty()) {
- headers.AddHeader(THttpInputHeader("If-None-Match", oldEtag));
- } else if (!oldLastModified.empty()) {
- headers.AddHeader(THttpInputHeader("If-Modified-Since", oldLastModified));
- }
- if (url.GetHost() == "yt.yandex-team.ru" || url.GetHost().EndsWith(".yt.yandex-team.ru")) {
- auto guid = CreateGuidAsString();
- headers.AddHeader(THttpInputHeader("X-YT-Correlation-Id", guid));
- YQL_LOG(INFO) << "Use Correlation-Id=" << guid << " for " << url.PrintS();
- }
-
- try {
- return Fetch(url, headers, TDuration::MilliSeconds(socketTimeoutMs));
- } catch (const std::exception& e) {
- // remap exception type to leverage retry logic
- throw TDownloadError() << e.what();
- }
- }
-
- static std::pair<ui64, TString> CopyToFile(const TString& url, IFetchResult& src, const TString& dstFile) {
- TFile outFile(dstFile, CreateAlways | ARW | AX);
- TUnbufferedFileOutput out(outFile);
- TMd5OutputStream md5Out(out);
-
- THttpInput& httpStream = src.GetStream();
- TDownloadStream input(httpStream);
- const ui64 size = TransferData(&input, &md5Out);
- auto result = std::make_pair(size, md5Out.Finalize());
- out.Finish();
- outFile.Close();
-
- ui64 contentLength = 0;
- // additional check for not compressed data
- if (!httpStream.ContentEncoded() && httpStream.GetContentLength(contentLength) && contentLength != size) {
- // let's retry this error
- ythrow TDownloadError() << "Size mismatch while downloading url " << url << ", downloaded size: " << size << ", ContentLength: " << contentLength;
- }
-
- if (auto trailers = httpStream.Trailers()) {
- if (auto header = trailers->FindHeader("X-YT-Error")) {
- ythrow TDownloadError() << "X-YT-Error=" << header->Value();
- }
- }
-
- i64 dstFileLen = GetFileLength(dstFile.c_str());
- if (dstFileLen == -1) {
- ythrow TSystemError() << "cannot get file length: " << dstFile;
- }
-
- YQL_ENSURE(static_cast<ui64>(dstFileLen) == size);
- return result;
- }
-
- static TString WeakETag2Strong(const TString& etag) {
- // drop W/ at the beginning if any
- return etag.StartsWith("W/") ? etag.substr(2) : etag;
- }
-
- static std::pair<TString, TString> ExtractETagAndLastModified(IFetchResult& result) {
- const auto& headers = result.GetStream().Headers();
- TString etag;
- TString lastModified;
- // linear scan
- for (auto it = headers.Begin(); it != headers.End(); ++it) {
- if (TCIEqualTo<TString>()(it->Name(), TString(TStringBuf("ETag")))) {
- etag = WeakETag2Strong(it->Value());
- }
-
- if (TCIEqualTo<TString>()(it->Name(), TString(TStringBuf("Last-Modified")))) {
- lastModified = it->Value();
- }
- }
-
- return std::make_pair(etag, lastModified);
- }
-
-private:
- ui32 SocketTimeoutMs;
- TPatternGroup AllowedUrls;
-};
-
-IFileStorage::IDownloaderPtr MakeHttpDownloader(bool restictedUser, const TFileStorageConfig& config, const std::vector<TString>& extraAllowedUrls) {
- return MakeIntrusive<THttpDownloader>(restictedUser, config, extraAllowedUrls);
-}
-
-} // NYql
+#include "http_download.h"
+#include "pattern_group.h"
+
+#include <ydb/library/yql/core/file_storage/proto/file_storage.pb.h>
+#include <ydb/library/yql/core/file_storage/download_stream.h>
+
+#include <ydb/library/yql/utils/fetch/fetch.h>
+#include <ydb/library/yql/utils/log/log.h>
+#include <ydb/library/yql/utils/log/context.h>
+#include <ydb/library/yql/utils/multi_resource_lock.h>
+#include <ydb/library/yql/utils/md5_stream.h>
+#include <ydb/library/yql/utils/retry.h>
+#include <ydb/library/yql/utils/yql_panic.h>
+
+#include <library/cpp/digest/md5/md5.h>
+#include <library/cpp/http/misc/httpcodes.h>
+
+#include <util/generic/guid.h>
+#include <util/generic/yexception.h>
+#include <util/stream/file.h>
+#include <util/system/file.h>
+
+
+namespace NYql {
+
+class THttpDownloader: public IFileStorage::IDownloader {
+public:
+ THttpDownloader(bool restictedUser, const TFileStorageConfig& config, const std::vector<TString>& extraAllowedUrls)
+ : SocketTimeoutMs(config.GetSocketTimeoutMs())
+ {
+ for (const auto& p : restictedUser ? config.GetExternalAllowedUrlPatterns() : config.GetAllowedUrlPatterns()) {
+ AllowedUrls.Add(p);
+ }
+ for (auto p: extraAllowedUrls) {
+ AllowedUrls.Add(p);
+ }
+ }
+ ~THttpDownloader() = default;
+
+ bool Accept(const THttpURL& url) final {
+ switch (url.GetScheme()) {
+ case NUri::TScheme::SchemeHTTP:
+ case NUri::TScheme::SchemeHTTPS:
+ return true;
+ default:
+ break;
+ }
+ return false;
+ }
+
+ std::tuple<TStorage::TDataPuller, TString, TString> Download(const THttpURL& url, const TString& oauthToken, const TString& oldEtag, const TString& oldLastModified) final {
+ if (!AllowedUrls.IsEmpty()) {
+ if (auto strUrl = url.PrintS(); !AllowedUrls.Match(strUrl)) {
+ YQL_LOG(WARN) << "FileStorage: url " << strUrl << " is not in whitelist, reject downloading";
+ throw yexception() << "It is not allowed to download url " << strUrl;
+ }
+ }
+
+ TFetchResultPtr fr1 = FetchWithETagAndLastModified(url, oauthToken, oldEtag, oldLastModified, SocketTimeoutMs);
+ switch (fr1->GetRetCode()) {
+ case HTTP_NOT_MODIFIED:
+ return std::make_tuple(TStorage::TDataPuller{}, TString{}, TString{});
+ case HTTP_OK:
+ break;
+ default:
+ ythrow yexception() << "Url " << url.PrintS() << " cannot be accessed, code: " << fr1->GetRetCode();
+ }
+
+ auto pair = ExtractETagAndLastModified(*fr1);
+
+ auto puller = [urlStr = url.PrintS(), fr1](const TFsPath& dstPath) -> std::pair<ui64, TString> {
+ return CopyToFile(urlStr, *fr1, dstPath);
+ };
+
+ return std::make_tuple(puller, pair.first, pair.second);
+ }
+
+private:
+ static TFetchResultPtr FetchWithETagAndLastModified(const THttpURL& url, const TString& oauthToken, const TString& oldEtag, const TString& oldLastModified, ui32 socketTimeoutMs) {
+ // more details about ETag and ModifiedSince: https://www.w3.org/Protocols/rfc2616/rfc2616-sec14.html#sec14.26
+ THttpHeaders headers;
+ if (!oauthToken.empty()) {
+ headers.AddHeader(THttpInputHeader("Authorization", "OAuth " + oauthToken));
+ }
+
+ // ETag has priority over modification time
+ if (!oldEtag.empty()) {
+ headers.AddHeader(THttpInputHeader("If-None-Match", oldEtag));
+ } else if (!oldLastModified.empty()) {
+ headers.AddHeader(THttpInputHeader("If-Modified-Since", oldLastModified));
+ }
+ if (url.GetHost() == "yt.yandex-team.ru" || url.GetHost().EndsWith(".yt.yandex-team.ru")) {
+ auto guid = CreateGuidAsString();
+ headers.AddHeader(THttpInputHeader("X-YT-Correlation-Id", guid));
+ YQL_LOG(INFO) << "Use Correlation-Id=" << guid << " for " << url.PrintS();
+ }
+
+ try {
+ return Fetch(url, headers, TDuration::MilliSeconds(socketTimeoutMs));
+ } catch (const std::exception& e) {
+ // remap exception type to leverage retry logic
+ throw TDownloadError() << e.what();
+ }
+ }
+
+ static std::pair<ui64, TString> CopyToFile(const TString& url, IFetchResult& src, const TString& dstFile) {
+ TFile outFile(dstFile, CreateAlways | ARW | AX);
+ TUnbufferedFileOutput out(outFile);
+ TMd5OutputStream md5Out(out);
+
+ THttpInput& httpStream = src.GetStream();
+ TDownloadStream input(httpStream);
+ const ui64 size = TransferData(&input, &md5Out);
+ auto result = std::make_pair(size, md5Out.Finalize());
+ out.Finish();
+ outFile.Close();
+
+ ui64 contentLength = 0;
+ // additional check for not compressed data
+ if (!httpStream.ContentEncoded() && httpStream.GetContentLength(contentLength) && contentLength != size) {
+ // let's retry this error
+ ythrow TDownloadError() << "Size mismatch while downloading url " << url << ", downloaded size: " << size << ", ContentLength: " << contentLength;
+ }
+
+ if (auto trailers = httpStream.Trailers()) {
+ if (auto header = trailers->FindHeader("X-YT-Error")) {
+ ythrow TDownloadError() << "X-YT-Error=" << header->Value();
+ }
+ }
+
+ i64 dstFileLen = GetFileLength(dstFile.c_str());
+ if (dstFileLen == -1) {
+ ythrow TSystemError() << "cannot get file length: " << dstFile;
+ }
+
+ YQL_ENSURE(static_cast<ui64>(dstFileLen) == size);
+ return result;
+ }
+
+ static TString WeakETag2Strong(const TString& etag) {
+ // drop W/ at the beginning if any
+ return etag.StartsWith("W/") ? etag.substr(2) : etag;
+ }
+
+ static std::pair<TString, TString> ExtractETagAndLastModified(IFetchResult& result) {
+ const auto& headers = result.GetStream().Headers();
+ TString etag;
+ TString lastModified;
+ // linear scan
+ for (auto it = headers.Begin(); it != headers.End(); ++it) {
+ if (TCIEqualTo<TString>()(it->Name(), TString(TStringBuf("ETag")))) {
+ etag = WeakETag2Strong(it->Value());
+ }
+
+ if (TCIEqualTo<TString>()(it->Name(), TString(TStringBuf("Last-Modified")))) {
+ lastModified = it->Value();
+ }
+ }
+
+ return std::make_pair(etag, lastModified);
+ }
+
+private:
+ ui32 SocketTimeoutMs;
+ TPatternGroup AllowedUrls;
+};
+
+IFileStorage::IDownloaderPtr MakeHttpDownloader(bool restictedUser, const TFileStorageConfig& config, const std::vector<TString>& extraAllowedUrls) {
+ return MakeIntrusive<THttpDownloader>(restictedUser, config, extraAllowedUrls);
+}
+
+} // NYql
diff --git a/ydb/library/yql/core/file_storage/http_download/http_download.h b/ydb/library/yql/core/file_storage/http_download/http_download.h
index a0abc1c111..827d212ada 100644
--- a/ydb/library/yql/core/file_storage/http_download/http_download.h
+++ b/ydb/library/yql/core/file_storage/http_download/http_download.h
@@ -1,13 +1,13 @@
-#pragma once
-
-#include <ydb/library/yql/core/file_storage/file_storage.h>
-
-#include <vector>
-
-namespace NYql {
-
-class TFileStorageConfig;
-
-IFileStorage::IDownloaderPtr MakeHttpDownloader(bool restictedUser, const TFileStorageConfig& config, const std::vector<TString>& extraAllowedUrls);
-
-} // NYql
+#pragma once
+
+#include <ydb/library/yql/core/file_storage/file_storage.h>
+
+#include <vector>
+
+namespace NYql {
+
+class TFileStorageConfig;
+
+IFileStorage::IDownloaderPtr MakeHttpDownloader(bool restictedUser, const TFileStorageConfig& config, const std::vector<TString>& extraAllowedUrls);
+
+} // NYql
diff --git a/ydb/library/yql/core/file_storage/http_download/pattern_group.cpp b/ydb/library/yql/core/file_storage/http_download/pattern_group.cpp
index f60ec107be..213ab9fcad 100644
--- a/ydb/library/yql/core/file_storage/http_download/pattern_group.cpp
+++ b/ydb/library/yql/core/file_storage/http_download/pattern_group.cpp
@@ -1,4 +1,4 @@
-#include <ydb/library/yql/core/file_storage/http_download/pattern_group.h>
+#include <ydb/library/yql/core/file_storage/http_download/pattern_group.h>
namespace NYql {
diff --git a/ydb/library/yql/core/file_storage/http_download/pattern_group_ut.cpp b/ydb/library/yql/core/file_storage/http_download/pattern_group_ut.cpp
index 1b917252c1..d5f2327c79 100644
--- a/ydb/library/yql/core/file_storage/http_download/pattern_group_ut.cpp
+++ b/ydb/library/yql/core/file_storage/http_download/pattern_group_ut.cpp
@@ -1,5 +1,5 @@
#include "pattern_group.h"
-
+
#include <library/cpp/testing/unittest/registar.h>
using namespace NYql;
diff --git a/ydb/library/yql/core/file_storage/http_download/ut/ya.make b/ydb/library/yql/core/file_storage/http_download/ut/ya.make
index fec8e3e8c5..9b17a31e01 100644
--- a/ydb/library/yql/core/file_storage/http_download/ut/ya.make
+++ b/ydb/library/yql/core/file_storage/http_download/ut/ya.make
@@ -1,9 +1,9 @@
-UNITTEST_FOR(ydb/library/yql/core/file_storage/http_download)
-
-OWNER(g:yql)
-
-SRCS(
- pattern_group_ut.cpp
-)
-
-END()
+UNITTEST_FOR(ydb/library/yql/core/file_storage/http_download)
+
+OWNER(g:yql)
+
+SRCS(
+ pattern_group_ut.cpp
+)
+
+END()
diff --git a/ydb/library/yql/core/file_storage/http_download/ya.make b/ydb/library/yql/core/file_storage/http_download/ya.make
index 344836558d..0d08930565 100644
--- a/ydb/library/yql/core/file_storage/http_download/ya.make
+++ b/ydb/library/yql/core/file_storage/http_download/ya.make
@@ -1,25 +1,25 @@
-LIBRARY()
-
-OWNER(g:yql)
-
-SRCS(
- http_download.cpp
- pattern_group.cpp
-)
-
-PEERDIR(
- ydb/library/yql/core/file_storage
- ydb/library/yql/core/file_storage/proto
- ydb/library/yql/utils/fetch
- ydb/library/yql/utils/log
- ydb/library/yql/utils
- library/cpp/regex/pcre
- library/cpp/digest/md5
- library/cpp/http/misc
-)
-
-END()
-
-RECURSE_FOR_TESTS(
- ut
-)
+LIBRARY()
+
+OWNER(g:yql)
+
+SRCS(
+ http_download.cpp
+ pattern_group.cpp
+)
+
+PEERDIR(
+ ydb/library/yql/core/file_storage
+ ydb/library/yql/core/file_storage/proto
+ ydb/library/yql/utils/fetch
+ ydb/library/yql/utils/log
+ ydb/library/yql/utils
+ library/cpp/regex/pcre
+ library/cpp/digest/md5
+ library/cpp/http/misc
+)
+
+END()
+
+RECURSE_FOR_TESTS(
+ ut
+)
diff --git a/ydb/library/yql/core/file_storage/proto/file_storage.proto b/ydb/library/yql/core/file_storage/proto/file_storage.proto
index c8fa642932..1f06481d1d 100644
--- a/ydb/library/yql/core/file_storage/proto/file_storage.proto
+++ b/ydb/library/yql/core/file_storage/proto/file_storage.proto
@@ -1,15 +1,15 @@
-package NYql;
-
-message TSchemeTranslate {
- required string Pattern = 1; // regexp to match
- required string TargetUrl = 2; // replacement string for target URL
-}
-
+package NYql;
+
+message TSchemeTranslate {
+ required string Pattern = 1; // regexp to match
+ required string TargetUrl = 2; // replacement string for target URL
+}
+
message TFileStorageConfig {
- optional string Path = 1; // Path to file storage. An auto-cleaned temp directory is used for empty value
- optional uint32 MaxFiles = 2 [default = 1000]; // Maximum number of files in the storage
- optional uint32 MaxSizeMb = 3 [default = 100]; // Maximum total size of all files in the storage
- repeated TSchemeTranslate CustomSchemes = 4;
+ optional string Path = 1; // Path to file storage. An auto-cleaned temp directory is used for empty value
+ optional uint32 MaxFiles = 2 [default = 1000]; // Maximum number of files in the storage
+ optional uint32 MaxSizeMb = 3 [default = 100]; // Maximum total size of all files in the storage
+ repeated TSchemeTranslate CustomSchemes = 4;
optional uint32 Threads = 5 [default = 1]; // Number of download threads for async downloading
optional uint32 RetryCount = 6 [default = 0]; // Number of additional attempts to download file
optional uint32 RetryDelayMs = 7 [default = 1000]; // Delay in ms between attempts to download file
@@ -19,4 +19,4 @@ message TFileStorageConfig {
optional string ArcSshUser = 10;
optional string ArcSshPkPath = 11;
optional string ArcTokenPath = 13;
-}
+}
diff --git a/ydb/library/yql/core/file_storage/proto/ya.make b/ydb/library/yql/core/file_storage/proto/ya.make
index addd85f79e..7dc1828aaf 100644
--- a/ydb/library/yql/core/file_storage/proto/ya.make
+++ b/ydb/library/yql/core/file_storage/proto/ya.make
@@ -1,11 +1,11 @@
OWNER(g:yql)
-
-PROTO_LIBRARY()
-
-SRCS(
- file_storage.proto
-)
-
+
+PROTO_LIBRARY()
+
+SRCS(
+ file_storage.proto
+)
+
EXCLUDE_TAGS(GO_PROTO)
-END()
+END()
diff --git a/ydb/library/yql/core/file_storage/sized_cache.cpp b/ydb/library/yql/core/file_storage/sized_cache.cpp
index fe17d895d0..91b8506246 100644
--- a/ydb/library/yql/core/file_storage/sized_cache.cpp
+++ b/ydb/library/yql/core/file_storage/sized_cache.cpp
@@ -1,98 +1,98 @@
-#include "sized_cache.h"
-
-#include <util/system/guard.h>
-
-namespace NYql {
-
-TSizedCache::TSizedCache(size_t maxEntries, ui64 maxSize)
- : Cache(Max<size_t>())
- , MaxEntries(maxEntries)
- , MaxSize(maxSize)
-{
+#include "sized_cache.h"
+
+#include <util/system/guard.h>
+
+namespace NYql {
+
+TSizedCache::TSizedCache(size_t maxEntries, ui64 maxSize)
+ : Cache(Max<size_t>())
+ , MaxEntries(maxEntries)
+ , MaxSize(maxSize)
+{
Y_ASSERT(MaxEntries > 0);
Y_ASSERT(MaxSize > 0);
-}
-
-void TSizedCache::Put(const TIntrusivePtr<ICacheObj>& obj, bool lock) {
+}
+
+void TSizedCache::Put(const TIntrusivePtr<ICacheObj>& obj, bool lock) {
const TString name = obj->GetName();
- const ui64 size = obj->GetSize();
-
- auto guard = Guard(Lock);
- auto it = Cache.Find(name);
- if (it == Cache.End()) {
- Cache.Insert(name, TEntry{obj, lock ? 1U : 0U});
- CurrentSize += size;
- } else if (lock) {
- ++it->Locks;
- }
- while (Cache.Size() > MaxEntries || CurrentSize > MaxSize) {
- it = Cache.FindOldest();
- if (Cache.End() == it || 0 != it->Locks) {
- break;
- }
- Remove(it);
- }
-}
-
+ const ui64 size = obj->GetSize();
+
+ auto guard = Guard(Lock);
+ auto it = Cache.Find(name);
+ if (it == Cache.End()) {
+ Cache.Insert(name, TEntry{obj, lock ? 1U : 0U});
+ CurrentSize += size;
+ } else if (lock) {
+ ++it->Locks;
+ }
+ while (Cache.Size() > MaxEntries || CurrentSize > MaxSize) {
+ it = Cache.FindOldest();
+ if (Cache.End() == it || 0 != it->Locks) {
+ break;
+ }
+ Remove(it);
+ }
+}
+
void TSizedCache::Release(const TString& name, bool remove) {
- auto guard = Guard(Lock);
- auto it = Cache.FindWithoutPromote(name);
- if (it == Cache.End()) {
+ auto guard = Guard(Lock);
+ auto it = Cache.FindWithoutPromote(name);
+ if (it == Cache.End()) {
Y_ASSERT(false);
- } else {
+ } else {
Y_ASSERT(it->Locks);
- if (it->Locks) {
- --it->Locks;
- }
- if (!it->Locks && (remove || Cache.Size() > MaxEntries || CurrentSize > MaxSize)) {
- Remove(it);
- }
- }
-}
-
+ if (it->Locks) {
+ --it->Locks;
+ }
+ if (!it->Locks && (remove || Cache.Size() > MaxEntries || CurrentSize > MaxSize)) {
+ Remove(it);
+ }
+ }
+}
+
TIntrusivePtr<TSizedCache::ICacheObj> TSizedCache::Get(const TString& name) {
- auto guard = Guard(Lock);
- auto it = Cache.Find(name);
- TIntrusivePtr<TSizedCache::ICacheObj> res;
- if (it != Cache.End()) {
- res = it->Obj;
- }
- return res;
-}
-
+ auto guard = Guard(Lock);
+ auto it = Cache.Find(name);
+ TIntrusivePtr<TSizedCache::ICacheObj> res;
+ if (it != Cache.End()) {
+ res = it->Obj;
+ }
+ return res;
+}
+
TIntrusivePtr<TSizedCache::ICacheObj> TSizedCache::Revoke(const TString& name) {
- auto guard = Guard(Lock);
- auto it = Cache.FindWithoutPromote(name);
- TIntrusivePtr<TSizedCache::ICacheObj> res;
- if (it != Cache.End()) {
- res = it->Obj;
+ auto guard = Guard(Lock);
+ auto it = Cache.FindWithoutPromote(name);
+ TIntrusivePtr<TSizedCache::ICacheObj> res;
+ if (it != Cache.End()) {
+ res = it->Obj;
Y_ASSERT(CurrentSize >= res->GetSize());
- CurrentSize -= res->GetSize();
- Cache.Erase(it);
- }
- return res;
-}
-
+ CurrentSize -= res->GetSize();
+ Cache.Erase(it);
+ }
+ return res;
+}
+
TMaybe<ui32> TSizedCache::GetLocks(const TString& name) const {
- auto guard = Guard(Lock);
- auto it = Cache.FindWithoutPromote(name);
- TMaybe<ui32> res;
- if (it != Cache.End()) {
- res = it->Locks;
- }
- return res;
-}
-
-void TSizedCache::Remove(TCache::TIterator it) {
- TIntrusivePtr<TSizedCache::ICacheObj> obj = it->Obj;
+ auto guard = Guard(Lock);
+ auto it = Cache.FindWithoutPromote(name);
+ TMaybe<ui32> res;
+ if (it != Cache.End()) {
+ res = it->Locks;
+ }
+ return res;
+}
+
+void TSizedCache::Remove(TCache::TIterator it) {
+ TIntrusivePtr<TSizedCache::ICacheObj> obj = it->Obj;
Y_ASSERT(CurrentSize >= obj->GetSize());
- CurrentSize -= obj->GetSize();
- Cache.Erase(it);
- // Dismissing may take a long time. Unlock temporary
- auto unguard = Unguard(Lock);
- obj->Dismiss();
-}
-
-} // NYql
-
-
+ CurrentSize -= obj->GetSize();
+ Cache.Erase(it);
+ // Dismissing may take a long time. Unlock temporary
+ auto unguard = Unguard(Lock);
+ obj->Dismiss();
+}
+
+} // NYql
+
+
diff --git a/ydb/library/yql/core/file_storage/sized_cache.h b/ydb/library/yql/core/file_storage/sized_cache.h
index a08095cab7..2d87aa7622 100644
--- a/ydb/library/yql/core/file_storage/sized_cache.h
+++ b/ydb/library/yql/core/file_storage/sized_cache.h
@@ -1,75 +1,75 @@
-#pragma once
-
+#pragma once
+
#include <library/cpp/cache/cache.h>
-
-#include <util/system/spinlock.h>
+
+#include <util/system/spinlock.h>
#include <util/generic/string.h>
-#include <util/generic/ylimits.h>
-#include <util/generic/ptr.h>
-#include <util/generic/maybe.h>
-
-namespace NYql {
-
-/*
- Cache with limits by count and by occupied size.
- All added objects must have unmodifiable size.
- The same object can be added multiple times. If it is added with lock then
- it should be Release()-ed the same number of times.
- Locked objects are not removed even if total count/size exceeds the limit.
-*/
-class TSizedCache {
-public:
- struct ICacheObj: public TThrRefBase {
- // Unique object identifier
+#include <util/generic/ylimits.h>
+#include <util/generic/ptr.h>
+#include <util/generic/maybe.h>
+
+namespace NYql {
+
+/*
+ Cache with limits by count and by occupied size.
+ All added objects must have unmodifiable size.
+ The same object can be added multiple times. If it is added with lock then
+ it should be Release()-ed the same number of times.
+ Locked objects are not removed even if total count/size exceeds the limit.
+*/
+class TSizedCache {
+public:
+ struct ICacheObj: public TThrRefBase {
+ // Unique object identifier
virtual TString GetName() = 0;
- // Object size
- virtual ui64 GetSize() = 0;
- // Called when object is removed from the cache. Should cleanup occupied resources
- virtual void Dismiss() = 0;
- };
-
-public:
- // Constructs the cache with the specified limits.
- // maxEntries and maxSize must be greater than zero
- TSizedCache(size_t maxEntries, ui64 maxSize);
- ~TSizedCache() = default;
-
- // Put an object to the cache. If lock = true the increments object lock count
- void Put(const TIntrusivePtr<ICacheObj>& obj, bool lock = true);
- // Decrements object lock count. If lock count becomes zero and total count/size exceeds the limit
- // then object is removed from the cache immediately
+ // Object size
+ virtual ui64 GetSize() = 0;
+ // Called when object is removed from the cache. Should cleanup occupied resources
+ virtual void Dismiss() = 0;
+ };
+
+public:
+ // Constructs the cache with the specified limits.
+ // maxEntries and maxSize must be greater than zero
+ TSizedCache(size_t maxEntries, ui64 maxSize);
+ ~TSizedCache() = default;
+
+ // Put an object to the cache. If lock = true the increments object lock count
+ void Put(const TIntrusivePtr<ICacheObj>& obj, bool lock = true);
+ // Decrements object lock count. If lock count becomes zero and total count/size exceeds the limit
+ // then object is removed from the cache immediately
void Release(const TString& name, bool remove = false);
- // Returns the object if it exists in the cache
+ // Returns the object if it exists in the cache
TIntrusivePtr<ICacheObj> Get(const TString& name);
- // Removes and returns the object if it exists in the cache
+ // Removes and returns the object if it exists in the cache
TIntrusivePtr<ICacheObj> Revoke(const TString& name);
- // Returns count of object locks, or empty if object doesn't belong to the cache
+ // Returns count of object locks, or empty if object doesn't belong to the cache
TMaybe<ui32> GetLocks(const TString& name) const;
- // Total size of all objects in the cache
- ui64 GetOccupiedSize() const {
- return CurrentSize;
- }
- // Total count of objects in the cache
- size_t GetCount() const {
- return Cache.Size();
- }
-private:
- struct TEntry {
- TIntrusivePtr<ICacheObj> Obj;
- ui32 Locks;
- };
+ // Total size of all objects in the cache
+ ui64 GetOccupiedSize() const {
+ return CurrentSize;
+ }
+ // Total count of objects in the cache
+ size_t GetCount() const {
+ return Cache.Size();
+ }
+private:
+ struct TEntry {
+ TIntrusivePtr<ICacheObj> Obj;
+ ui32 Locks;
+ };
using TCache = TLRUCache<TString, TEntry>;
-
- void Remove(TCache::TIterator it);
-
-private:
- TCache Cache;
- size_t MaxEntries = 0;
- ui64 MaxSize = 0;
- ui64 CurrentSize = 0;
- TAdaptiveLock Lock;
-};
-
-} // NYql
-
-
+
+ void Remove(TCache::TIterator it);
+
+private:
+ TCache Cache;
+ size_t MaxEntries = 0;
+ ui64 MaxSize = 0;
+ ui64 CurrentSize = 0;
+ TAdaptiveLock Lock;
+};
+
+} // NYql
+
+
diff --git a/ydb/library/yql/core/file_storage/sized_cache_ut.cpp b/ydb/library/yql/core/file_storage/sized_cache_ut.cpp
index b0e41e5c62..915e8a1862 100644
--- a/ydb/library/yql/core/file_storage/sized_cache_ut.cpp
+++ b/ydb/library/yql/core/file_storage/sized_cache_ut.cpp
@@ -1,158 +1,158 @@
-#include "sized_cache.h"
-
+#include "sized_cache.h"
+
#include <library/cpp/testing/unittest/registar.h>
-
-using namespace NYql;
-
-struct TTestCacheObj: public TSizedCache::ICacheObj {
+
+using namespace NYql;
+
+struct TTestCacheObj: public TSizedCache::ICacheObj {
TTestCacheObj(const TString& name, ui64 size)
- : Name(name)
- , Size(size)
- , Dismissed(false)
- {
- }
-
+ : Name(name)
+ , Size(size)
+ , Dismissed(false)
+ {
+ }
+
TString GetName() override {
- return Name;
- }
+ return Name;
+ }
ui64 GetSize() override {
- return Size;
- }
+ return Size;
+ }
void Dismiss() override {
- Dismissed = true;
- }
-
+ Dismissed = true;
+ }
+
TString Name;
- ui64 Size;
- bool Dismissed;
-};
-
+ ui64 Size;
+ bool Dismissed;
+};
+
Y_UNIT_TEST_SUITE(TSizedCacheTests) {
Y_UNIT_TEST(Count) {
- TSizedCache cache(100, 100);
- TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
- TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
- TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
-
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 0);
- cache.Put(o1, false);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 1);
- cache.Put(o2, false);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 2);
- cache.Put(o3, false);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
- cache.Put(o1, false); // Equal object
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
- cache.Put(o3, false); // Equal object
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
- }
-
+ TSizedCache cache(100, 100);
+ TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
+ TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
+ TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
+
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 0);
+ cache.Put(o1, false);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 1);
+ cache.Put(o2, false);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 2);
+ cache.Put(o3, false);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
+ cache.Put(o1, false); // Equal object
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
+ cache.Put(o3, false); // Equal object
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
+ }
+
Y_UNIT_TEST(GetOccupiedSize) {
- TSizedCache cache(100, 100);
- TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
- TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
- TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
-
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 0);
- cache.Put(o1, false);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 10);
- cache.Put(o2, false);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 30);
- cache.Put(o3, false);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 45);
- cache.Put(o2, false); // Equla object
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 45);
- cache.Put(o3, false); // Equla object
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 45);
- }
-
+ TSizedCache cache(100, 100);
+ TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
+ TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
+ TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
+
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 0);
+ cache.Put(o1, false);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 10);
+ cache.Put(o2, false);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 30);
+ cache.Put(o3, false);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 45);
+ cache.Put(o2, false); // Equla object
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 45);
+ cache.Put(o3, false); // Equla object
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 45);
+ }
+
Y_UNIT_TEST(DisplaceByCount) {
- TSizedCache cache(2, 100);
- TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
- TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
- TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
-
- cache.Put(o1, false);
- cache.Put(o2, false);
- cache.Put(o3, false);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 2);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 35);
- UNIT_ASSERT(o1->Dismissed);
- UNIT_ASSERT(!o2->Dismissed);
- UNIT_ASSERT(!o3->Dismissed);
- }
-
+ TSizedCache cache(2, 100);
+ TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
+ TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
+ TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
+
+ cache.Put(o1, false);
+ cache.Put(o2, false);
+ cache.Put(o3, false);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 35);
+ UNIT_ASSERT(o1->Dismissed);
+ UNIT_ASSERT(!o2->Dismissed);
+ UNIT_ASSERT(!o3->Dismissed);
+ }
+
Y_UNIT_TEST(DisplaceBySize) {
- TSizedCache cache(100, 35);
- TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
- TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
- TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
-
- cache.Put(o1, false);
- cache.Put(o2, false);
- cache.Put(o3, false);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 2);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 35);
- UNIT_ASSERT(o1->Dismissed);
- UNIT_ASSERT(!o2->Dismissed);
- UNIT_ASSERT(!o3->Dismissed);
- }
-
+ TSizedCache cache(100, 35);
+ TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
+ TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
+ TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
+
+ cache.Put(o1, false);
+ cache.Put(o2, false);
+ cache.Put(o3, false);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 35);
+ UNIT_ASSERT(o1->Dismissed);
+ UNIT_ASSERT(!o2->Dismissed);
+ UNIT_ASSERT(!o3->Dismissed);
+ }
+
Y_UNIT_TEST(Lock) {
- TSizedCache cache(2, 30);
- TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
- TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
- TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
-
- UNIT_ASSERT(cache.GetLocks("o1").Empty());
- cache.Put(o1, true);
- UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 1);
- cache.Put(o2, true);
- cache.Put(o3, true);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 45);
- UNIT_ASSERT(!o1->Dismissed);
- UNIT_ASSERT(!o2->Dismissed);
- UNIT_ASSERT(!o3->Dismissed);
- cache.Release("o1"); // Unlock object. It will be removed immediately
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 2);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 35);
- UNIT_ASSERT(o1->Dismissed);
- UNIT_ASSERT(!o2->Dismissed);
- UNIT_ASSERT(!o3->Dismissed);
- }
-
+ TSizedCache cache(2, 30);
+ TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
+ TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
+ TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
+
+ UNIT_ASSERT(cache.GetLocks("o1").Empty());
+ cache.Put(o1, true);
+ UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 1);
+ cache.Put(o2, true);
+ cache.Put(o3, true);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 45);
+ UNIT_ASSERT(!o1->Dismissed);
+ UNIT_ASSERT(!o2->Dismissed);
+ UNIT_ASSERT(!o3->Dismissed);
+ cache.Release("o1"); // Unlock object. It will be removed immediately
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 35);
+ UNIT_ASSERT(o1->Dismissed);
+ UNIT_ASSERT(!o2->Dismissed);
+ UNIT_ASSERT(!o3->Dismissed);
+ }
+
Y_UNIT_TEST(MultiLock) {
- TSizedCache cache(2, 30);
- TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
- TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
- TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
-
- cache.Put(o1, true);
- UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 1);
- cache.Put(o1, true);
- UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 2);
- cache.Put(o1, true);
- UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 3);
- cache.Put(o2, true);
- cache.Put(o3, true);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 45);
- UNIT_ASSERT(!o1->Dismissed);
- UNIT_ASSERT(!o2->Dismissed);
- UNIT_ASSERT(!o3->Dismissed);
- cache.Release("o1");
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
- UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 2);
- cache.Release("o1");
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
- UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 1);
- cache.Release("o1"); // Unlock object. It will be removed immediately
- UNIT_ASSERT(cache.GetLocks("o1").Empty());
- UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 2);
- UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 35);
- UNIT_ASSERT(o1->Dismissed);
- UNIT_ASSERT(!o2->Dismissed);
- UNIT_ASSERT(!o3->Dismissed);
- }
-}
+ TSizedCache cache(2, 30);
+ TIntrusivePtr<TTestCacheObj> o1 = MakeIntrusive<TTestCacheObj>("o1", 10ull);
+ TIntrusivePtr<TTestCacheObj> o2 = MakeIntrusive<TTestCacheObj>("o2", 20ull);
+ TIntrusivePtr<TTestCacheObj> o3 = MakeIntrusive<TTestCacheObj>("o3", 15ull);
+
+ cache.Put(o1, true);
+ UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 1);
+ cache.Put(o1, true);
+ UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 2);
+ cache.Put(o1, true);
+ UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 3);
+ cache.Put(o2, true);
+ cache.Put(o3, true);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 45);
+ UNIT_ASSERT(!o1->Dismissed);
+ UNIT_ASSERT(!o2->Dismissed);
+ UNIT_ASSERT(!o3->Dismissed);
+ cache.Release("o1");
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 2);
+ cache.Release("o1");
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 3);
+ UNIT_ASSERT_VALUES_EQUAL(*cache.GetLocks("o1"), 1);
+ cache.Release("o1"); // Unlock object. It will be removed immediately
+ UNIT_ASSERT(cache.GetLocks("o1").Empty());
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetCount(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(cache.GetOccupiedSize(), 35);
+ UNIT_ASSERT(o1->Dismissed);
+ UNIT_ASSERT(!o2->Dismissed);
+ UNIT_ASSERT(!o3->Dismissed);
+ }
+}
diff --git a/ydb/library/yql/core/file_storage/storage.cpp b/ydb/library/yql/core/file_storage/storage.cpp
index 247bff5242..656e49ac67 100644
--- a/ydb/library/yql/core/file_storage/storage.cpp
+++ b/ydb/library/yql/core/file_storage/storage.cpp
@@ -1,38 +1,38 @@
-#include "storage.h"
-
+#include "storage.h"
+
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/rand_guid.h>
#include <ydb/library/yql/utils/proc_alive.h>
-
+
#include <library/cpp/digest/md5/md5.h>
#include <util/folder/dirut.h>
#include <util/generic/algorithm.h>
#include <util/generic/vector.h>
#include <util/generic/yexception.h>
-#include <util/generic/ptr.h>
+#include <util/generic/ptr.h>
#include <util/generic/utility.h>
-#include <util/system/file.h>
+#include <util/system/file.h>
#include <util/system/file_lock.h>
#include <util/system/fs.h>
-#include <util/system/maxlen.h>
+#include <util/system/maxlen.h>
#include <util/system/mutex.h>
#include <util/system/utime.h>
-#include <util/system/thread.h>
-
-#include <functional>
-
-#if defined(_unix_)
-#include <pthread.h>
-#endif
-
+#include <util/system/thread.h>
+
+#include <functional>
+
+#if defined(_unix_)
+#include <pthread.h>
+#endif
+
#include <errno.h>
-namespace NYql {
-
-namespace {
-
+namespace NYql {
+
+namespace {
+
struct TFileObject {
TString Name;
time_t MTime;
@@ -48,33 +48,33 @@ TFsPath ToFilePath(const TString& path)
return tempDir;
}
return path;
-}
-
+}
+
} // namespace
-
+
TFileLink::TFileLink(const TFsPath& path, const TString& storageFileName, ui64 size, const TString& md5, bool deleteOnDestroy)
- : Path(path)
- , StorageFileName(storageFileName)
- , Size(size)
+ : Path(path)
+ , StorageFileName(storageFileName)
+ , Size(size)
, Md5(md5)
, DeleteOnDestroy(deleteOnDestroy)
-{
-}
-
-TFileLink::~TFileLink() {
+{
+}
+
+TFileLink::~TFileLink() {
if (!DeleteOnDestroy) {
return;
}
- YQL_LOG(INFO) << "Destroying TFileLink for " << Path.GetPath().Quote();
+ YQL_LOG(INFO) << "Destroying TFileLink for " << Path.GetPath().Quote();
try {
Path.ForceDelete();
} catch (...) {
YQL_LOG(ERROR) << CurrentExceptionMessage();
}
-}
-
-TFileLinkPtr CreateFakeFileLink(const TFsPath& path, const TString& md5, bool deleteOnDestroy) {
+}
+
+TFileLinkPtr CreateFakeFileLink(const TFsPath& path, const TString& md5, bool deleteOnDestroy) {
if (!path.Exists()) {
ythrow yexception() << "Unable to create file link for non-existent file " << path.GetPath().Quote();
}
@@ -88,7 +88,7 @@ TFileLinkPtr CreateFakeFileLink(const TFsPath& path, const TString& md5, bool de
ythrow yexception() << "Unable to get size for file " << path.GetPath().Quote();
}
- return new TFileLink(path, effectiveMd5, size, effectiveMd5, deleteOnDestroy);
+ return new TFileLink(path, effectiveMd5, size, effectiveMd5, deleteOnDestroy);
}
bool SetCacheFilePermissionsNoThrow(const TString& path) {
@@ -106,60 +106,60 @@ void SetFilePermissions(const TString& path, int mode) {
}
}
-class TStorage::TImpl: public TIntrusiveListItem<TImpl> {
-public:
- class TAtforkReinit {
- public:
- inline TAtforkReinit() {
-#if defined(_bionic_)
-//no pthread_atfork on android libc
-#elif defined(_unix_)
- pthread_atfork(nullptr, nullptr, ProcessReinit);
-#endif
- }
-
- inline void Register(TImpl* obj) {
- auto guard = Guard(Mutex);
- Registered.PushBack(obj);
- }
-
- inline void Unregister(TImpl* obj) {
- auto guard = Guard(Mutex);
- obj->Unlink();
- }
-
- static TAtforkReinit& Get() {
- return *SingletonWithPriority<TAtforkReinit, 256>();
- }
-
- private:
- void Reinit() {
- with_lock (Mutex) {
+class TStorage::TImpl: public TIntrusiveListItem<TImpl> {
+public:
+ class TAtforkReinit {
+ public:
+ inline TAtforkReinit() {
+#if defined(_bionic_)
+//no pthread_atfork on android libc
+#elif defined(_unix_)
+ pthread_atfork(nullptr, nullptr, ProcessReinit);
+#endif
+ }
+
+ inline void Register(TImpl* obj) {
+ auto guard = Guard(Mutex);
+ Registered.PushBack(obj);
+ }
+
+ inline void Unregister(TImpl* obj) {
+ auto guard = Guard(Mutex);
+ obj->Unlink();
+ }
+
+ static TAtforkReinit& Get() {
+ return *SingletonWithPriority<TAtforkReinit, 256>();
+ }
+
+ private:
+ void Reinit() {
+ with_lock (Mutex) {
for (auto& v : Registered) {
v.ResetRandom();
- }
- }
- }
-
- static void ProcessReinit() {
- Get().Reinit();
- }
-
- TIntrusiveList<TImpl> Registered;
- TMutex Mutex;
- };
-
+ }
+ }
+ }
+
+ static void ProcessReinit() {
+ Get().Reinit();
+ }
+
+ TIntrusiveList<TImpl> Registered;
+ TMutex Mutex;
+ };
+
TImpl(size_t maxFiles, ui64 maxSize, const TString& storagePath)
: StorageDir(ToFilePath(storagePath))
, ProcessTempDir(StorageDir / ToString(GetPID())) // must be subfolder for fast hardlinking
, IsTemp(storagePath.empty())
, MaxFiles(maxFiles)
, MaxSize(maxSize)
- {
- // TFsPath is not thread safe. It can initialize internal Split at any time. Force do it right now
+ {
+ // TFsPath is not thread safe. It can initialize internal Split at any time. Force do it right now
StorageDir.PathSplit();
ProcessTempDir.PathSplit();
-
+
StorageDir.MkDirs(MODE0711);
ProcessTempDir.MkDirs(MODE0711);
#ifdef _linux_
@@ -171,33 +171,33 @@ public:
if (!IsTemp) {
LoadStats();
}
- TAtforkReinit::Get().Register(this);
+ TAtforkReinit::Get().Register(this);
YQL_LOG(INFO) << "FileStorage initialized in " << StorageDir.GetPath().Quote()
- << ", temporary dir: " << ProcessTempDir.GetPath().Quote()
- << ", files: " << CurrentFiles
+ << ", temporary dir: " << ProcessTempDir.GetPath().Quote()
+ << ", files: " << CurrentFiles
<< ", total size: " << CurrentSize;
- }
-
+ }
+
~TImpl() {
- TAtforkReinit::Get().Unregister(this);
- try {
+ TAtforkReinit::Get().Unregister(this);
+ try {
ProcessTempDir.ForceDelete();
- if (IsTemp) {
+ if (IsTemp) {
StorageDir.ForceDelete();
- }
- } catch (...) {
+ }
+ } catch (...) {
YQL_LOG(ERROR) << CurrentExceptionMessage();
- }
- }
-
+ }
+ }
+
const TFsPath& GetRoot() const {
return StorageDir;
- }
-
- const TFsPath& GetTemp() const {
- return ProcessTempDir;
- }
-
+ }
+
+ const TFsPath& GetTemp() const {
+ return ProcessTempDir;
+ }
+
TFileLinkPtr Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const TStorage::TDataPuller& puller) {
bool newFileAdded = false;
TFileLinkPtr result = HardlinkFromStorage(storageFileName, md5, outFileName);
@@ -208,18 +208,18 @@ public:
ui64 fileSize = 0;
TString pullerMd5; // overrides input arg 'md5'
- try {
- std::tie(fileSize, pullerMd5) = puller(hardlinkFile);
- } catch (...) {
- YQL_LOG(ERROR) << CurrentExceptionMessage();
- NFs::Remove(hardlinkFile);
- throw;
- }
+ try {
+ std::tie(fileSize, pullerMd5) = puller(hardlinkFile);
+ } catch (...) {
+ YQL_LOG(ERROR) << CurrentExceptionMessage();
+ NFs::Remove(hardlinkFile);
+ throw;
+ }
Y_ENSURE(hardlinkFile.Exists(), "FileStorage: cannot put not existing temporary path");
Y_ENSURE(hardlinkFile.IsFile(), "FileStorage: cannot put non-file temporary path");
- SetCacheFilePermissionsNoThrow(hardlinkFile);
-
+ SetCacheFilePermissionsNoThrow(hardlinkFile);
+
if (NFs::HardLink(hardlinkFile, storageFile)) {
AtomicIncrement(CurrentFiles);
AtomicAdd(CurrentSize, fileSize);
@@ -234,13 +234,13 @@ public:
YQL_LOG(INFO) << "Using " << (newFileAdded ? "new" : "existing") << " storage file " << result->GetStorageFileName().Quote()
<< ", temp path: " << result->GetPath().GetPath().Quote()
<< ", size: " << result->GetSize();
-
- if (newFileAdded) {
+
+ if (newFileAdded) {
Cleanup();
}
return result;
- }
-
+ }
+
TFileLinkPtr HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName) {
TFsPath storageFile = StorageDir / existingStorageFileName;
TFsPath hardlinkFile = ProcessTempDir / (outFileName ? outFileName : GetTempName());
@@ -275,7 +275,7 @@ public:
if (!NFs::Rename(src, dstStorageFile)) {
ythrow yexception() << "Failed to rename file from " << src << " to " << dstStorageFile;
}
- SetCacheFilePermissionsNoThrow(dstStorageFile);
+ SetCacheFilePermissionsNoThrow(dstStorageFile);
const i64 newFileSize = Max<i64>(0, GetFileLength(dstStorageFile.c_str()));
@@ -308,12 +308,12 @@ public:
ui64 GetOccupiedSize() const {
return AtomicGet(CurrentSize);
- }
-
+ }
+
size_t GetCount() const {
return AtomicGet(CurrentFiles);
- }
-
+ }
+
TString GetTempName() {
with_lock(RndLock) {
return Rnd.GenGuid();
@@ -334,12 +334,12 @@ private:
TFsPath childPath(StorageDir / name);
TFileStat stat(childPath, true);
if (stat.IsFile()) {
- ++actualFiles;
+ ++actualFiles;
actualSize += stat.Size;
} else if (stat.IsDir() && TryFromString(name, oldPid)) {
if (!IsProcessAlive(oldPid)) {
// cleanup of previously not cleaned hardlinks directory
- try {
+ try {
#ifdef _linux_
TFileLock childLock(childPath / ".lockfile");
TTryGuard guard(childLock);
@@ -352,28 +352,28 @@ private:
YQL_LOG(WARN) << "Not cleaning dead process dir " << childPath
<< ": " << "directory is still locked, skipping";
}
- } catch (...) {
- YQL_LOG(WARN) << "Error cleaning dead process dir " << childPath
- << ": " << CurrentExceptionMessage();
- }
+ } catch (...) {
+ YQL_LOG(WARN) << "Error cleaning dead process dir " << childPath
+ << ": " << CurrentExceptionMessage();
+ }
}
}
}
CurrentFiles = actualFiles;
CurrentSize = actualSize;
- }
-
+ }
+
bool NeedToCleanup() {
return static_cast<ui64>(AtomicGet(CurrentFiles)) > MaxFiles ||
static_cast<ui64>(AtomicGet(CurrentSize)) > MaxSize;
- }
-
+ }
+
void Cleanup() {
- if (!NeedToCleanup()) {
- return;
- }
-
+ if (!NeedToCleanup()) {
+ return;
+ }
+
with_lock (CleanupLock) {
TVector<TString> names;
StorageDir.ListNames(names);
@@ -389,15 +389,15 @@ private:
TFileStat stat(childPath, true);
if (stat.IsFile()) {
files.push_back(TFileObject{name, stat.MTime, stat.Size});
- ++actualFiles;
+ ++actualFiles;
actualSize += stat.Size;
- }
- }
+ }
+ }
// sort files to get older files first
Sort(files, [](const TFileObject& f1, const TFileObject& f2) {
if (f1.MTime == f2.MTime) {
- return f1.Name.compare(f2.Name) < 0;
+ return f1.Name.compare(f2.Name) < 0;
}
return f1.MTime < f2.MTime;
});
@@ -413,26 +413,26 @@ private:
YQL_LOG(INFO) << "Removing file from cache (name: " << f.Name
<< ", size: " << f.Size
<< ", mtime: " << f.MTime << ")";
- if (!NFs::Remove(StorageDir / f.Name)) {
- YQL_LOG(WARN) << "Failed to remove file " << f.Name.Quote() << ": " << LastSystemErrorText();
- } else {
- --actualFiles;
- actualSize -= f.Size;
- }
+ if (!NFs::Remove(StorageDir / f.Name)) {
+ YQL_LOG(WARN) << "Failed to remove file " << f.Name.Quote() << ": " << LastSystemErrorText();
+ } else {
+ --actualFiles;
+ actualSize -= f.Size;
+ }
}
AtomicSet(CurrentFiles, actualFiles);
AtomicSet(CurrentSize, actualSize);
- }
- }
-
- void ResetRandom() {
+ }
+ }
+
+ void ResetRandom() {
with_lock(RndLock) {
Rnd.ResetSeed();
}
- }
-
-private:
+ }
+
+private:
TMutex CleanupLock;
const TFsPath StorageDir;
const TFsPath ProcessTempDir;
@@ -442,33 +442,33 @@ private:
const ui64 MaxSize;
TAtomic CurrentFiles = 0;
TAtomic CurrentSize = 0;
- TMutex RndLock;
+ TMutex RndLock;
TRandGuid Rnd;
-};
-
+};
+
TStorage::TStorage(size_t maxFiles, ui64 maxSize, const TString& storagePath)
: Impl(new TImpl(maxFiles, maxSize, storagePath))
+{
+}
+
+TStorage::~TStorage()
{
}
-TStorage::~TStorage()
-{
-}
-
-TFsPath TStorage::GetRoot() const
-{
- return Impl->GetRoot();
-}
-
-TFsPath TStorage::GetTemp() const
-{
- return Impl->GetTemp();
-}
+TFsPath TStorage::GetRoot() const
+{
+ return Impl->GetRoot();
+}
+TFsPath TStorage::GetTemp() const
+{
+ return Impl->GetTemp();
+}
+
TFileLinkPtr TStorage::Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const TDataPuller& puller)
-{
+{
return Impl->Put(storageFileName, outFileName, md5, puller);
-}
+}
TFileLinkPtr TStorage::HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName)
{
@@ -485,18 +485,18 @@ bool TStorage::RemoveFromStorage(const TString& existingStorageFileName)
return Impl->RemoveFromStorage(existingStorageFileName);
}
-ui64 TStorage::GetOccupiedSize() const
-{
- return Impl->GetOccupiedSize();
-}
-
-size_t TStorage::GetCount() const
-{
- return Impl->GetCount();
-}
-
+ui64 TStorage::GetOccupiedSize() const
+{
+ return Impl->GetOccupiedSize();
+}
+
+size_t TStorage::GetCount() const
+{
+ return Impl->GetCount();
+}
+
TString TStorage::GetTempName()
{
return Impl->GetTempName();
}
-} // NYql
+} // NYql
diff --git a/ydb/library/yql/core/file_storage/storage.h b/ydb/library/yql/core/file_storage/storage.h
index 8ab42a69a1..e6aa55b35a 100644
--- a/ydb/library/yql/core/file_storage/storage.h
+++ b/ydb/library/yql/core/file_storage/storage.h
@@ -1,18 +1,18 @@
-#pragma once
-
-#include <util/folder/path.h>
-#include <util/generic/ptr.h>
+#pragma once
+
+#include <util/folder/path.h>
+#include <util/generic/ptr.h>
#include <util/generic/string.h>
-#include <util/generic/maybe.h>
-
-#include <functional>
-
-namespace NYql {
-
+#include <util/generic/maybe.h>
+
+#include <functional>
+
+namespace NYql {
+
class TFileLink: public TThrRefBase {
public:
TFileLink(const TFsPath& path, const TString& storageFileName, ui64 size, const TString& md5, bool deleteOnDestroy = true);
- ~TFileLink();
+ ~TFileLink();
const TString& GetStorageFileName() const { return StorageFileName; }
const TFsPath& GetPath() const { return Path; }
@@ -29,53 +29,53 @@ private:
using TFileLinkPtr = TIntrusivePtr<TFileLink>;
-TFileLinkPtr CreateFakeFileLink(const TFsPath& path, const TString& md5 = "", bool deleteOnDestroy = false);
+TFileLinkPtr CreateFakeFileLink(const TFsPath& path, const TString& md5 = "", bool deleteOnDestroy = false);
-/* File cache with limits by count and by occupied size.
+/* File cache with limits by count and by occupied size.
The same file can be added multiple times.
It is safe to add the same file from multiple threads/processes simultaneously.
Only first thread will do the actual work. Other threads will wait and reuse the result.
-*/
-class TStorage {
-public:
- // This function is called by the storage to transfer user data to the provided temporary file path
+*/
+class TStorage {
+public:
+ // This function is called by the storage to transfer user data to the provided temporary file path
// Returns content size and md5
using TDataPuller = std::function<std::pair<ui64, TString> (const TFsPath& dstPath)>;
- /* Constructs the storage with the specified limits.
- storagePath can be empty - a temporary directory will be used in this case.
- */
+ /* Constructs the storage with the specified limits.
+ storagePath can be empty - a temporary directory will be used in this case.
+ */
TStorage(size_t maxFiles, ui64 maxSize, const TString& storagePath = {});
- ~TStorage();
+ ~TStorage();
- // Returns root storage directory
- TFsPath GetRoot() const;
- // Returns temp storage directory
- TFsPath GetTemp() const;
- // Puts the passed data to the storage with the specified storage file name.
+ // Returns root storage directory
+ TFsPath GetRoot() const;
+ // Returns temp storage directory
+ TFsPath GetTemp() const;
+ // Puts the passed data to the storage with the specified storage file name.
// The second argument outFileName specifies a name of temporary link returned from the Put(). If empty, then random guid is used.
// Provide valid md5 if it is known in advance, otherwise pass "". It will be overridden by puller result
- // The provided puller does the actual transfer of the data to a storage file. It can take a long time.
+ // The provided puller does the actual transfer of the data to a storage file. It can take a long time.
TFileLinkPtr Put(const TString& storageFileName, const TString& outFileName, const TString& md5, const TDataPuller& puller);
// Returns nullptr on error
TFileLinkPtr HardlinkFromStorage(const TString& existingStorageFileName, const TString& storageFileMd5, const TString& outFileName);
void MoveToStorage(const TFsPath& src, const TString& dstStorageFileName);
bool RemoveFromStorage(const TString& existingStorageFileName);
- // Total size of all files in the cache
- ui64 GetOccupiedSize() const;
- // Total count of files in the cache
- size_t GetCount() const;
+ // Total size of all files in the cache
+ ui64 GetOccupiedSize() const;
+ // Total count of files in the cache
+ size_t GetCount() const;
TString GetTempName();
-private:
+private:
class TImpl;
- THolder<TImpl> Impl;
-};
-
+ THolder<TImpl> Impl;
+};
+
constexpr int MODE0711 = S_IRWXU | S_IXGRP | S_IXOTH;
constexpr int MODE0744 = S_IRWXU | S_IRGRP | S_IROTH;
void SetCacheFilePermissions(const TString& path);
bool SetCacheFilePermissionsNoThrow(const TString& path);
void SetFilePermissions(const TString& path, int mode);
-} // NYql
+} // NYql
diff --git a/ydb/library/yql/core/file_storage/storage_ut.cpp b/ydb/library/yql/core/file_storage/storage_ut.cpp
index 4229abd219..f895f13246 100644
--- a/ydb/library/yql/core/file_storage/storage_ut.cpp
+++ b/ydb/library/yql/core/file_storage/storage_ut.cpp
@@ -1,67 +1,67 @@
-#include "storage.h"
-
+#include "storage.h"
+
#include <library/cpp/testing/unittest/registar.h>
#include <library/cpp/threading/future/async.h>
-
+
#include <util/datetime/base.h>
-#include <util/folder/path.h>
+#include <util/folder/path.h>
#include <util/generic/algorithm.h>
-#include <util/generic/yexception.h>
-#include <util/stream/file.h>
+#include <util/generic/yexception.h>
+#include <util/stream/file.h>
#include <util/stream/input.h>
-#include <util/stream/str.h>
-
-using namespace NYql;
-using namespace NThreading;
-
+#include <util/stream/str.h>
+
+using namespace NYql;
+using namespace NThreading;
+
static TString DATA = "1234567890";
static TString DATA_MD5 = "e807f1fcf82d132f9bb018ca6738a19f";
-
+
Y_UNIT_TEST_SUITE(TStorageTests) {
-
- class TTestDir {
- private:
- TFsPath Path_;
-
- public:
+
+ class TTestDir {
+ private:
+ TFsPath Path_;
+
+ public:
TTestDir(const TString& name) {
- Y_ENSURE(name.length() > 0, "have to specify name");
+ Y_ENSURE(name.length() > 0, "have to specify name");
Y_ENSURE(name.find('.') == TString::npos, "must be simple name");
Y_ENSURE(name.find('/') == TString::npos, "must be simple name");
Y_ENSURE(name.find('\\') == TString::npos, "must be simple name");
- Path_ = TFsPath(name);
-
- Path_.ForceDelete();
- Path_.MkDir();
- }
- ~TTestDir() {
- Path_.ForceDelete();
- }
-
- TFsPath GetFsPath() const {
- return Path_;
- }
- };
-
+ Path_ = TFsPath(name);
+
+ Path_.ForceDelete();
+ Path_.MkDir();
+ }
+ ~TTestDir() {
+ Path_.ForceDelete();
+ }
+
+ TFsPath GetFsPath() const {
+ return Path_;
+ }
+ };
+
TFileLinkPtr PutFile(const TString& name, TStorage& storage, const TString& tmpOutName = {}, int sleepSeconds = 0) {
return storage.Put(name, tmpOutName, DATA_MD5, [&](const TFsPath& storagePath) {
Sleep(TDuration::Seconds(sleepSeconds));
- TStringInput in(DATA);
+ TStringInput in(DATA);
TUnbufferedFileOutput out(storagePath);
return std::make_pair(TransferData(&in, &out), DATA_MD5);
- });
- }
-
+ });
+ }
+
TFileLinkPtr PutFileWithSleep(const TString& name, TStorage& storage) {
return PutFile(name, storage, {}, 1);
- }
-
+ }
+
Y_UNIT_TEST(Put) {
THolder<TStorage> storage = MakeHolder<TStorage>(10, 100);
- TFileLinkPtr fileInStorage = PutFile("test.file", *storage, "somename");
+ TFileLinkPtr fileInStorage = PutFile("test.file", *storage, "somename");
TFsPath rootPath(fileInStorage->GetPath());
UNIT_ASSERT(rootPath.Exists());
- UNIT_ASSERT_VALUES_EQUAL(rootPath.GetName(), "somename");
+ UNIT_ASSERT_VALUES_EQUAL(rootPath.GetName(), "somename");
UNIT_ASSERT_VALUES_EQUAL(fileInStorage->GetStorageFileName(), "test.file");
UNIT_ASSERT(fileInStorage->GetPath().IsSubpathOf(storage->GetRoot().Fix()));
UNIT_ASSERT_VALUES_EQUAL(TIFStream(fileInStorage->GetPath()).ReadAll(), DATA);
@@ -69,22 +69,22 @@ Y_UNIT_TEST_SUITE(TStorageTests) {
UNIT_ASSERT_EQUAL(storage->GetOccupiedSize(), DATA.size());
fileInStorage.Reset();
UNIT_ASSERT(false == rootPath.Exists());
- }
-
+ }
+
Y_UNIT_TEST(ParallelPut) {
THolder<TStorage> storage = MakeHolder<TStorage>(10, 100);
TThreadPool queue;
- queue.Start(10);
-
+ queue.Start(10);
+
TVector<TFuture<TFileLinkPtr>> res;
- res.reserve(10);
- for (size_t i = 0; i < 10; ++i) {
+ res.reserve(10);
+ for (size_t i = 0; i < 10; ++i) {
res.emplace_back(Async([&]() {
return PutFileWithSleep("test.file", *storage);
}, queue));
- }
- for (auto& f: res) {
- f.Wait();
+ }
+ for (auto& f: res) {
+ f.Wait();
TFileLinkPtr fileInStorage = f.GetValue();
UNIT_ASSERT(fileInStorage->GetPath().Exists());
UNIT_ASSERT_VALUES_EQUAL(fileInStorage->GetStorageFileName(), "test.file");
@@ -92,31 +92,31 @@ Y_UNIT_TEST_SUITE(TStorageTests) {
UNIT_ASSERT_VALUES_EQUAL(TIFStream(fileInStorage->GetPath()).ReadAll(), DATA);
UNIT_ASSERT_EQUAL(storage->GetCount(), 1);
UNIT_ASSERT_EQUAL(storage->GetOccupiedSize(), DATA.size());
- }
- }
-
+ }
+ }
+
Y_UNIT_TEST(CleanUp) {
THolder<TStorage> storage = MakeHolder<TStorage>(10, 100);
TFileLinkPtr fileInStorage = PutFile("test.file", *storage);
UNIT_ASSERT(fileInStorage->GetPath().Exists());
-
- auto rootPath = storage->GetRoot();
- storage.Destroy();
+
+ auto rootPath = storage->GetRoot();
+ storage.Destroy();
UNIT_ASSERT(!fileInStorage->GetPath().Exists());
- UNIT_ASSERT(!rootPath.Exists());
- }
-
+ UNIT_ASSERT(!rootPath.Exists());
+ }
+
Y_UNIT_TEST(DisplaceByCount) {
ui64 maxCount = 2;
THolder<TStorage> storage = MakeHolder<TStorage>(maxCount, 100);
-
+
TFileLinkPtr file1 = PutFile("test1.file", *storage);
TFileLinkPtr file2 = PutFile("test2.file", *storage);
- UNIT_ASSERT_VALUES_EQUAL(storage->GetCount(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(storage->GetCount(), 2);
UNIT_ASSERT_VALUES_EQUAL(storage->GetOccupiedSize(), 2 * DATA.size());
UNIT_ASSERT(file1->GetPath().Exists());
UNIT_ASSERT(file2->GetPath().Exists());
-
+
TFileLinkPtr file3 = PutFileWithSleep("test3.file", *storage);
// after putting 3rd file we exceeded the limit by count
@@ -141,61 +141,61 @@ Y_UNIT_TEST_SUITE(TStorageTests) {
UNIT_ASSERT(Find(beg, end, file1->GetStorageFileName()) == end);
UNIT_ASSERT(Find(beg, end, file2->GetStorageFileName()) == end);
UNIT_ASSERT(Find(beg, end, file3->GetStorageFileName()) != end);
- }
-
+ }
+
Y_UNIT_TEST(DisplaceBySize) {
ui64 maxSize = 25;
THolder<TStorage> storage = MakeHolder<TStorage>(10, maxSize);
-
+
TFileLinkPtr file1 = PutFile("test1.file", *storage);
TFileLinkPtr file2 = PutFile("test2.file", *storage);
- UNIT_ASSERT_VALUES_EQUAL(storage->GetCount(), 2);
+ UNIT_ASSERT_VALUES_EQUAL(storage->GetCount(), 2);
UNIT_ASSERT_VALUES_EQUAL(storage->GetOccupiedSize(), 2 * DATA.size());
UNIT_ASSERT(file1->GetPath().Exists());
UNIT_ASSERT(file2->GetPath().Exists());
-
+
TFileLinkPtr file3 = PutFileWithSleep("test3.file", *storage);
-
+
// after putting 3rd file we exceeded the limit by size
// and cleaned up files till we get half of the original maxSize
UNIT_ASSERT_VALUES_EQUAL(storage->GetCount(), 1);
UNIT_ASSERT_VALUES_EQUAL(storage->GetOccupiedSize(), DATA.size());
-
+
// despite of file1 and file2 are displaced from storage, we still
// have temporary hardlinks to it
UNIT_ASSERT(file1->GetPath().Exists());
UNIT_ASSERT(file2->GetPath().Exists());
UNIT_ASSERT(file3->GetPath().Exists());
-
+
TVector<TString> filesInStorage;
storage->GetRoot().ListNames(filesInStorage);
UNIT_ASSERT_EQUAL(filesInStorage.size(), 2); // 1 file + 1 hardlink directory
-
+
auto beg = filesInStorage.begin(),
end = filesInStorage.end();
-
+
// file1 and file2 were displaced
UNIT_ASSERT(Find(beg, end, file1->GetStorageFileName()) == end);
UNIT_ASSERT(Find(beg, end, file2->GetStorageFileName()) == end);
UNIT_ASSERT(Find(beg, end, file3->GetStorageFileName()) != end);
- }
-
+ }
+
Y_UNIT_TEST(PersistStorage) {
- TTestDir dir("PersistStorage");
+ TTestDir dir("PersistStorage");
THolder<TStorage> storage = MakeHolder<TStorage>(100, 100, dir.GetFsPath());
- auto rootPath = storage->GetRoot();
-
+ auto rootPath = storage->GetRoot();
+
TFileLinkPtr fileInStorage = PutFile("test.file", *storage);
UNIT_ASSERT_EQUAL(storage->GetCount(), 1);
UNIT_ASSERT_EQUAL(storage->GetOccupiedSize(), DATA.size());
- storage.Destroy();
+ storage.Destroy();
UNIT_ASSERT(!fileInStorage->GetPath().Exists()); // hardlink was deleted
- UNIT_ASSERT(rootPath.Exists());
+ UNIT_ASSERT(rootPath.Exists());
UNIT_ASSERT((rootPath / fileInStorage->GetStorageFileName()).Exists());
-
- storage.Reset(new TStorage(100, 100, dir.GetFsPath()));
+
+ storage.Reset(new TStorage(100, 100, dir.GetFsPath()));
UNIT_ASSERT_EQUAL(storage->GetCount(), 1);
UNIT_ASSERT_EQUAL(storage->GetOccupiedSize(), DATA.size());
- }
-}
+ }
+}
diff --git a/ydb/library/yql/core/file_storage/url_mapper_ut.cpp b/ydb/library/yql/core/file_storage/url_mapper_ut.cpp
index 287caf054c..cbcf73abd2 100644
--- a/ydb/library/yql/core/file_storage/url_mapper_ut.cpp
+++ b/ydb/library/yql/core/file_storage/url_mapper_ut.cpp
@@ -19,11 +19,11 @@ Y_UNIT_TEST_SUITE(TUrlMapperTests) {
m.AddMapping("yt://([a-zA-Z0-9\\-_]+)/(.+)@t=([0-9A-Fa-f]+\\-[0-9A-Fa-f]+\\-[0-9A-Fa-f]+\\-[0-9A-Fa-f]+)", "https://$1.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$2&transaction_id=$3");
m.AddMapping("yt://([a-zA-Z0-9\\-_]+)/(.+)", "https://$1.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$2");
- m.AddMapping("https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/#page=navigation&path=//(.+)", "https://$2.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$3");
- m.AddMapping("https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/navigation\\?path=//(.+)", "https://$2.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$3");
-
- m.AddMapping("https?://a\\.yandex-team\\.ru/arc/(.+)/arcadia/(.+)\\?rev=(\\d+)", "arc://$2?rev=$3&branch=$1");
+ m.AddMapping("https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/#page=navigation&path=//(.+)", "https://$2.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$3");
+ m.AddMapping("https?://yt\\.yandex(\\.net|-team\\.ru)/([a-zA-Z0-9\\-_]+)/navigation\\?path=//(.+)", "https://$2.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//$3");
+ m.AddMapping("https?://a\\.yandex-team\\.ru/arc/(.+)/arcadia/(.+)\\?rev=(\\d+)", "arc://$2?rev=$3&branch=$1");
+
TString tmp;
UNIT_ASSERT(!m.MapUrl("http://ya.ru", tmp));
UNIT_ASSERT(!m.MapUrl("http://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//home", tmp));
@@ -42,16 +42,16 @@ Y_UNIT_TEST_SUITE(TUrlMapperTests) {
// transaction guid with 3 groups instead of 4
RemapAndCheck(m, "yt://hahn/statbox/home/nile/tmp/abc.txt@t=9797-164f64-3fe0190", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//statbox/home/nile/tmp/abc.txt@t=9797-164f64-3fe0190");
RemapAndCheck(m, "https://yt.yandex-team.ru/hahn/#page=navigation&path=//home/yql/dev/1.txt", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//home/yql/dev/1.txt");
- RemapAndCheck(m, "https://yt.yandex-team.ru/hahn/navigation?path=//home/yql/dev/1.txt", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//home/yql/dev/1.txt");
+ RemapAndCheck(m, "https://yt.yandex-team.ru/hahn/navigation?path=//home/yql/dev/1.txt", "https://hahn.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//home/yql/dev/1.txt");
// with dash
RemapAndCheck(m, "yt://seneca-sas/tmp/bsistat02i_202624_1523866684_inclusion_already_moderating", "https://seneca-sas.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//tmp/bsistat02i_202624_1523866684_inclusion_already_moderating");
// with underscore and digits
RemapAndCheck(m, "yt://seneca_sas0123456789/tmp/bsistat02i_202624_1523866684_inclusion_already_moderating", "https://seneca_sas0123456789.yt.yandex-team.ru/api/v3/read_file?disposition=attachment&path=//tmp/bsistat02i_202624_1523866684_inclusion_already_moderating");
-
- RemapAndCheck(m, "https://a.yandex-team.ru/arc/trunk/arcadia/yql/ya.make?rev=5530789", "arc://yql/ya.make?rev=5530789&branch=trunk");
- RemapAndCheck(m, "https://a.yandex-team.ru/arc/branches/yql/yql-stable-2019-08-16/arcadia/yql/ya.make?rev=5530789", "arc://yql/ya.make?rev=5530789&branch=branches/yql/yql-stable-2019-08-16");
-
- UNIT_ASSERT(!m.MapUrl("https://a.yandex-team.ru/arc/trunk/arcadia/yql/ya.make", tmp));
+
+ RemapAndCheck(m, "https://a.yandex-team.ru/arc/trunk/arcadia/yql/ya.make?rev=5530789", "arc://yql/ya.make?rev=5530789&branch=trunk");
+ RemapAndCheck(m, "https://a.yandex-team.ru/arc/branches/yql/yql-stable-2019-08-16/arcadia/yql/ya.make?rev=5530789", "arc://yql/ya.make?rev=5530789&branch=branches/yql/yql-stable-2019-08-16");
+
+ UNIT_ASSERT(!m.MapUrl("https://a.yandex-team.ru/arc/trunk/arcadia/yql/ya.make", tmp));
}
}
diff --git a/ydb/library/yql/core/file_storage/ut/ya.make b/ydb/library/yql/core/file_storage/ut/ya.make
index c19f58203b..b2bc5fb8cc 100644
--- a/ydb/library/yql/core/file_storage/ut/ya.make
+++ b/ydb/library/yql/core/file_storage/ut/ya.make
@@ -1,7 +1,7 @@
UNITTEST_FOR(ydb/library/yql/core/file_storage)
-
+
OWNER(g:yql)
-
+
IF (OPENSOURCE)
CFLAGS(
-DOPENSOURCE
@@ -9,18 +9,18 @@ IF (OPENSOURCE)
ENDIF()
-SRCS(
+SRCS(
file_storage_ut.cpp
- sized_cache_ut.cpp
- storage_ut.cpp
+ sized_cache_ut.cpp
+ storage_ut.cpp
test_http_server.cpp
url_mapper_ut.cpp
-)
-
+)
+
PEERDIR(
library/cpp/http/server
library/cpp/threading/future
- ydb/library/yql/core/file_storage/http_download
+ ydb/library/yql/core/file_storage/http_download
)
-END()
+END()
diff --git a/ydb/library/yql/core/file_storage/ya.make b/ydb/library/yql/core/file_storage/ya.make
index f25592bfb7..0cef760097 100644
--- a/ydb/library/yql/core/file_storage/ya.make
+++ b/ydb/library/yql/core/file_storage/ya.make
@@ -1,13 +1,13 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:yql)
-
-SRCS(
+
+SRCS(
download_stream.cpp
download_stream.h
- file_storage.cpp
+ file_storage.cpp
file_storage.h
- sized_cache.cpp
+ sized_cache.cpp
sized_cache.h
storage.cpp
storage.h
@@ -15,22 +15,22 @@ SRCS(
url_mapper.h
url_meta.cpp
url_meta.h
-)
-
-PEERDIR(
+)
+
+PEERDIR(
library/cpp/cache
library/cpp/digest/md5
library/cpp/logger/global
- library/cpp/threading/future
+ library/cpp/threading/future
library/cpp/regex/pcre
- library/cpp/uri
+ library/cpp/uri
ydb/library/yql/core/file_storage/proto
ydb/library/yql/utils
- ydb/library/yql/utils/log
+ ydb/library/yql/utils/log
ydb/library/yql/utils/fetch
-)
-
-END()
+)
+
+END()
RECURSE_FOR_TESTS(
ut
diff --git a/ydb/library/yql/core/issue/protos/issue_id.proto b/ydb/library/yql/core/issue/protos/issue_id.proto
index a2ed91d640..87d6048410 100644
--- a/ydb/library/yql/core/issue/protos/issue_id.proto
+++ b/ydb/library/yql/core/issue/protos/issue_id.proto
@@ -22,15 +22,15 @@ message TIssuesIds {
CORE_OPTIMIZATION = 1070;
CORE_PARAM_EVALUATION = 1080;
CORE_EXPR_EVALUATION = 1081;
- CORE_POST_TYPE_ANN = 1090;
+ CORE_POST_TYPE_ANN = 1090;
// core warnings
CORE_CAST_YSON_JSON_BYTES = 1100;
CORE_FREE_TABLE_PATH_RECORD = 1101;
CORE_CAST_INTEGRAL_TO_TIMESTAMP_UNSAFE = 1102;
- CORE_FREE_KEY_SWITCH = 1103;
+ CORE_FREE_KEY_SWITCH = 1103;
CORE_NON_PERSISTABLE_ENTITY = 1104;
- CORE_NON_STREAM_BATCH_UDF = 1105;
+ CORE_NON_STREAM_BATCH_UDF = 1105;
CORE_FLATTEN_BY_OPT = 1106;
CORE_IMPLICIT_BITCAST = 1107;
CORE_LEGACY_IN_FOR_EMPTY_OR_NULLABLE = 1108;
@@ -86,18 +86,18 @@ message TIssuesIds {
YT_INFER_SCHEMA = 3002;
YT_ACCESS_DENIED = 3003;
YT_TABLE_NOT_FOUND = 3004;
- YT_NATIVE_PRAGMA = 3005;
+ YT_NATIVE_PRAGMA = 3005;
YT_VIEW_NOT_FOUND = 3006;
YT_SCHEMELESS_TABLE = 3007;
YT_ENTRY_NOT_TABLE_OR_VIEW = 3008;
- YT_CONCURRENT_TABLE_MODIF = 3009;
- YT_LATE_TABLE_XLOCK = 3010;
- YT_SORT_ORDER_CHANGE = 3011;
+ YT_CONCURRENT_TABLE_MODIF = 3009;
+ YT_LATE_TABLE_XLOCK = 3010;
+ YT_SORT_ORDER_CHANGE = 3011;
YT_MAX_DATAWEIGHT_PER_JOB_EXCEEDED = 3012;
- YT_WARN_TABLE_DOES_NOT_EXIST = 3013;
- YT_ROWSPEC_HIDES_FIELDS = 3014;
- YT_ROWSPEC_DIFF_SORT = 3015;
- YT_NATIVE_DESC_SORT_IGNORED = 3016;
+ YT_WARN_TABLE_DOES_NOT_EXIST = 3013;
+ YT_ROWSPEC_HIDES_FIELDS = 3014;
+ YT_ROWSPEC_DIFF_SORT = 3015;
+ YT_NATIVE_DESC_SORT_IGNORED = 3016;
// yql parser warnings
YQL_PRAGMA_WARNING_MSG = 4500;
@@ -118,7 +118,7 @@ message TIssuesIds {
YQL_DEPRECATED_V0_SYNTAX = 4515;
YQL_UNNAMED_COLUMN = 4516;
YQL_SOURCE_SELECT_COLUMN_MISMATCH = 4517;
- YQL_DEPRECATED_PRAGMA = 4518;
+ YQL_DEPRECATED_PRAGMA = 4518;
YQL_EMPTY_WINDOW_FRAME = 4520;
YQL_RANK_WITHOUT_ORDER_BY = 4521;
YQL_LIMIT_ORDER_BY_WITH_UNION = 4522;
@@ -134,9 +134,9 @@ message TIssuesIds {
YQL_PROJECTION_ALIAS_IS_REFERENCED_IN_GROUP_BY = 4532;
YQL_TABLE_BINDING_DUPLICATE = 4533;
-// yql parser errors
- YQL_NOT_ALLOWED_IN_DISCOVERY = 4600;
-
+// yql parser errors
+ YQL_NOT_ALLOWED_IN_DISCOVERY = 4600;
+
// jsonpath errors
JSONPATH_PARSE_ERROR = 4700;
JSONPATH_EXPECTED_OBJECT = 4701;
diff --git a/ydb/library/yql/core/issue/yql_issue.txt b/ydb/library/yql/core/issue/yql_issue.txt
index 897d5d6f9a..ddb4531484 100644
--- a/ydb/library/yql/core/issue/yql_issue.txt
+++ b/ydb/library/yql/core/issue/yql_issue.txt
@@ -37,11 +37,11 @@ ids {
format: "Type annotation"
}
ids {
- code: CORE_POST_TYPE_ANN
- severity: S_INFO
- format: "Post type annotation"
-}
-ids {
+ code: CORE_POST_TYPE_ANN
+ severity: S_INFO
+ format: "Post type annotation"
+}
+ids {
code: CORE_INTENT
severity: S_INFO
format: "Table intent determination"
@@ -84,18 +84,18 @@ ids {
severity: S_WARNING
}
ids {
- code: CORE_FREE_KEY_SWITCH
- severity: S_WARNING
-}
-ids {
+ code: CORE_FREE_KEY_SWITCH
+ severity: S_WARNING
+}
+ids {
code: CORE_NON_PERSISTABLE_ENTITY
severity: S_WARNING
}
ids {
- code: CORE_NON_STREAM_BATCH_UDF
- severity: S_WARNING
-}
-ids {
+ code: CORE_NON_STREAM_BATCH_UDF
+ severity: S_WARNING
+}
+ids {
code: CORE_FLATTEN_BY_OPT
severity: S_WARNING
}
@@ -333,14 +333,14 @@ ids {
severity: S_WARNING
}
ids {
- code: YQL_DEPRECATED_PRAGMA
- severity: S_WARNING
-}
-ids {
- code: YT_TABLE_PATH_RECORD_FOR_TMP
- severity: S_WARNING
-}
-ids {
+ code: YQL_DEPRECATED_PRAGMA
+ severity: S_WARNING
+}
+ids {
+ code: YT_TABLE_PATH_RECORD_FOR_TMP
+ severity: S_WARNING
+}
+ids {
code: YQL_DEPRECATED_DATETIME2
severity: S_WARNING
}
@@ -349,22 +349,22 @@ ids {
severity: S_WARNING
}
ids {
- code: YT_INFER_SCHEMA
- severity: S_ERROR
-}
-ids {
- code: YT_ACCESS_DENIED
- severity: S_ERROR
-}
-ids {
- code: YT_TABLE_NOT_FOUND
- severity: S_ERROR
-}
-ids {
- code: YT_NATIVE_PRAGMA
- severity: S_WARNING
-}
-ids {
+ code: YT_INFER_SCHEMA
+ severity: S_ERROR
+}
+ids {
+ code: YT_ACCESS_DENIED
+ severity: S_ERROR
+}
+ids {
+ code: YT_TABLE_NOT_FOUND
+ severity: S_ERROR
+}
+ids {
+ code: YT_NATIVE_PRAGMA
+ severity: S_WARNING
+}
+ids {
code: YT_VIEW_NOT_FOUND
severity: S_ERROR
}
@@ -377,38 +377,38 @@ ids {
severity: S_ERROR
}
ids {
- code: YT_CONCURRENT_TABLE_MODIF
- severity: S_ERROR
-}
-ids {
- code: YT_LATE_TABLE_XLOCK
- severity: S_WARNING
-}
-ids {
- code: YT_SORT_ORDER_CHANGE
- severity: S_WARNING
-}
-ids {
+ code: YT_CONCURRENT_TABLE_MODIF
+ severity: S_ERROR
+}
+ids {
+ code: YT_LATE_TABLE_XLOCK
+ severity: S_WARNING
+}
+ids {
+ code: YT_SORT_ORDER_CHANGE
+ severity: S_WARNING
+}
+ids {
code: YT_MAX_DATAWEIGHT_PER_JOB_EXCEEDED
severity: S_ERROR
}
ids {
- code: YT_WARN_TABLE_DOES_NOT_EXIST
- severity: S_WARNING
-}
-ids {
- code: YT_ROWSPEC_HIDES_FIELDS
- severity: S_WARNING
-}
-ids {
- code: YT_ROWSPEC_DIFF_SORT
- severity: S_WARNING
-}
-ids {
- code: YT_NATIVE_DESC_SORT_IGNORED
- severity: S_WARNING
-}
-ids {
+ code: YT_WARN_TABLE_DOES_NOT_EXIST
+ severity: S_WARNING
+}
+ids {
+ code: YT_ROWSPEC_HIDES_FIELDS
+ severity: S_WARNING
+}
+ids {
+ code: YT_ROWSPEC_DIFF_SORT
+ severity: S_WARNING
+}
+ids {
+ code: YT_NATIVE_DESC_SORT_IGNORED
+ severity: S_WARNING
+}
+ids {
code: STAT_DEPRECATED_STRING_TREE
severity: S_WARNING
}
@@ -439,10 +439,10 @@ ids {
ids {
code: YQL_SOURCE_SELECT_COLUMN_MISMATCH
severity: S_WARNING
-}
-ids {
- code: YQL_NOT_ALLOWED_IN_DISCOVERY
- severity: S_ERROR
+}
+ids {
+ code: YQL_NOT_ALLOWED_IN_DISCOVERY
+ severity: S_ERROR
}
ids {
code: JSONPATH_PARSE_ERROR
diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp
index ce22b12571..60a21bff41 100644
--- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp
+++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.cpp
@@ -17,7 +17,7 @@
#include <ydb/library/yql/core/services/yql_transform_pipeline.h>
#include <ydb/library/yql/core/services/yql_out_transformers.h>
#include <ydb/library/yql/utils/yql_paths.h>
-
+
#include <util/generic/xrange.h>
#include <library/cpp/yson/writer.h>
@@ -44,7 +44,7 @@ template <typename T> TString ToLiteral(const T value) { return ToString<T>(valu
template <typename TRandomType>
TExprNode::TPtr Random0Arg(const TExprNode::TPtr& node, TExprContext& ctx, TTypeAnnotationContext& types) {
if (node->ChildrenSize() == 0U) {
- YQL_CLOG(DEBUG, CorePeepHole) << "0-arg " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "0-arg " << node->Content();
const auto random = types.GetCachedRandom<TRandomType>();
return ctx.NewCallable(node->Pos(), node->GetTypeAnn()->Cast<TDataExprType>()->GetName(), { ctx.NewAtom(node->Pos(), ToLiteral(random)) });
}
@@ -54,7 +54,7 @@ TExprNode::TPtr Random0Arg(const TExprNode::TPtr& node, TExprContext& ctx, TType
template <ui64(*Convert)(ui64)>
TExprNode::TPtr Now0Arg(const TExprNode::TPtr& node, TExprContext& ctx, TTypeAnnotationContext& types) {
if (node->ChildrenSize() == 0U) {
- YQL_CLOG(DEBUG, CorePeepHole) << "0-arg " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "0-arg " << node->Content();
const auto now = types.GetCachedNow();
return ctx.NewCallable(node->Pos(), node->GetTypeAnn()->Cast<TDataExprType>()->GetName(), { ctx.NewAtom(node->Pos(), ToString(bool(Convert) ? Convert(now) : now), TNodeFlags::Default) });
}
@@ -446,7 +446,7 @@ TExprNode::TPtr PeepHoleConvertGroupBySingleKey(const TExprNode::TPtr& node, TEx
return node;
}
- YQL_CLOG(DEBUG, CorePeepHole) << "Convert " << node->Content() << " single key";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Convert " << node->Content() << " single key";
const auto handlerLambda = node->Child(2);
const auto& keyArg = handlerLambda->Head().Head();
const auto& listArg = handlerLambda->Head().Tail();
@@ -457,10 +457,10 @@ TExprNode::TPtr PeepHoleConvertGroupBySingleKey(const TExprNode::TPtr& node, TEx
TExprNode::TPtr PeepHolePlainKeyForPartitionByKey(const TExprNode::TPtr& node, TExprContext& ctx) {
auto keySelectorLambda = node->Child(1);
bool needPickle = false;
- if (RemoveOptionalType(keySelectorLambda->GetTypeAnn())->GetKind() == ETypeAnnotationKind::Data) {
+ if (RemoveOptionalType(keySelectorLambda->GetTypeAnn())->GetKind() == ETypeAnnotationKind::Data) {
// ok
- } else if (keySelectorLambda->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
- auto tupleType = keySelectorLambda->GetTypeAnn()->Cast<TTupleExprType>();
+ } else if (keySelectorLambda->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
+ auto tupleType = keySelectorLambda->GetTypeAnn()->Cast<TTupleExprType>();
if (tupleType->GetSize() < 2) {
needPickle = true;
} else {
@@ -479,7 +479,7 @@ TExprNode::TPtr PeepHolePlainKeyForPartitionByKey(const TExprNode::TPtr& node, T
return node;
}
- YQL_CLOG(DEBUG, CorePeepHole) << "Plain key for " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Plain key for " << node->Content();
return ctx.Builder(node->Pos())
.Callable(node->Content())
.Add(0, node->HeadPtr())
@@ -501,7 +501,7 @@ TExprNode::TPtr PeepHolePlainKeyForPartitionByKey(const TExprNode::TPtr& node, T
.Param("item")
.List()
.Callable(0, "Unpickle")
- .Add(0, ExpandType(keySelectorLambda->Pos(), *keySelectorLambda->GetTypeAnn(), ctx))
+ .Add(0, ExpandType(keySelectorLambda->Pos(), *keySelectorLambda->GetTypeAnn(), ctx))
.Callable(1, "Nth")
.Arg(0, "item")
.Atom(1, "0", TNodeFlags::Default)
@@ -522,7 +522,7 @@ TExprNode::TPtr PeepHolePlainKeyForPartitionByKey(const TExprNode::TPtr& node, T
}
TExprNode::TPtr PeepHoleExpandExtractItems(const TExprNode::TPtr& node, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
const TCoExtractMembers extract(node);
const auto& input = extract.Input();
@@ -543,13 +543,13 @@ TExprNode::TPtr PeepHoleExpandExtractItems(const TExprNode::TPtr& node, TExprCon
auto body = ctx.NewCallable(extract.Pos(), "AsStruct", std::move(fields));
auto lambda = ctx.NewLambda(extract.Pos(), ctx.NewArguments(extract.Pos(), { std::move(arg) }), std::move(body));
- const bool ordered = input.Ref().GetConstraint<TSortedConstraintNode>();
+ const bool ordered = input.Ref().GetConstraint<TSortedConstraintNode>();
return ctx.Builder(extract.Pos())
- .Callable(ordered ? "OrderedMap" : "Map")
- .Add(0, input.Ptr())
+ .Callable(ordered ? "OrderedMap" : "Map")
+ .Add(0, input.Ptr())
.Add(1, std::move(lambda))
- .Seal()
- .Build();
+ .Seal()
+ .Build();
}
TExprNode::TPtr PeepHoleDictFromKeysToDict(const TExprNode::TPtr& node, TExprContext& ctx) {
@@ -580,7 +580,7 @@ TExprNode::TPtr PeepHoleDictFromKeysToDict(const TExprNode::TPtr& node, TExprCon
.Build()
.PayloadSelector()
.Args({"item"})
- .Body<TCoVoid>().Build()
+ .Body<TCoVoid>().Build()
.Build()
.Settings()
.Add().Build("One")
@@ -606,7 +606,7 @@ TExprNode::TPtr ExpandCastOverData(const TExprNode::TPtr& input, TExprContext& c
auto to = input->GetTypeAnn()->Cast<TDataExprType>();
auto ret = input;
if constexpr (Strong) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content()
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content()
<< " for " << *input->Head().GetTypeAnn()
<< " to " << *input->GetTypeAnn();
ret = ctx.RenameNode(*input, "SafeCast");
@@ -632,7 +632,7 @@ TExprNode::TPtr ExpandCastOverData(const TExprNode::TPtr& input, TExprContext& c
template <bool Strong>
TExprNode::TPtr ExpandCastOverOptionalData(const TExprNode::TPtr& input, TExprContext& ctx) {
if constexpr (Strong) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content()
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content()
<< " for " << *input->Head().GetTypeAnn()
<< " to " << *input->GetTypeAnn();
@@ -678,7 +678,7 @@ TExprNode::TPtr ExpandCastOverOptional(const TExprNode::TPtr& input, TExprContex
const auto targetItemType = targetType->Cast<TOptionalExprType>()->GetItemType();
if (ETypeAnnotationKind::Null == targetItemType->GetKind()) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Null?";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Null?";
return ctx.Builder(input->Pos())
.Callable("If")
.Callable(0, "Exists")
@@ -696,7 +696,7 @@ TExprNode::TPtr ExpandCastOverOptional(const TExprNode::TPtr& input, TExprContex
const auto targetLevel = GetOptionalLevel(targetItemType);
if (opt && targetLevel > 0U) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content();
auto stub = ExpandType(input->Tail().Pos(), *targetType, ctx);
auto type = ExpandType(input->Tail().Pos(), *targetItemType, ctx);
@@ -759,7 +759,7 @@ TExprNode::TPtr ExpandCastOverOptional(const TExprNode::TPtr& input, TExprContex
const bool flat = opt || sourceLevel > targetLevel;
auto type = ExpandType(input->Tail().Pos(), flat ? *targetType : *targetItemType, ctx);
if (!opt && sourceLevel < targetLevel) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " as Just";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " as Just";
return ctx.Builder(input->Pos())
.Callable("Just")
.Callable(0, input->Content())
@@ -770,7 +770,7 @@ TExprNode::TPtr ExpandCastOverOptional(const TExprNode::TPtr& input, TExprContex
}
const auto worker = flat ? "FlatMap" : "Map";
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " as " << worker;
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " as " << worker;
return ctx.Builder(input->Pos())
.Callable(worker)
.Add(0, input->HeadPtr())
@@ -786,7 +786,7 @@ TExprNode::TPtr ExpandCastOverOptional(const TExprNode::TPtr& input, TExprContex
template <bool Strong>
TExprNode::TPtr ExpandCastOverList(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for List";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for List";
const auto sourceType = input->Head().GetTypeAnn();
const auto targetType = input->GetTypeAnn();
const auto sourceItemType = sourceType->Cast<TListExprType>()->GetItemType();
@@ -806,31 +806,31 @@ TExprNode::TPtr ExpandCastOverList(const TExprNode::TPtr& input, TExprContext& c
.Seal().Build();
}
-template <bool Strong, class TSeqType>
-TExprNode::TPtr ExpandCastOverSequence(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for sequence";
- const auto sourceType = input->Head().GetTypeAnn();
- const auto targetType = input->GetTypeAnn();
- const auto sourceItemType = sourceType->Cast<TSeqType>()->GetItemType();
- const auto targetItemType = targetType->Cast<TSeqType>()->GetItemType();
- const bool opt = CastMayFail<Strong>(sourceItemType, targetItemType);
- auto type = ExpandType(input->Tail().Pos(), opt ? *ctx.MakeType<TOptionalExprType>(targetItemType) : *targetItemType, ctx);
- return ctx.Builder(input->Pos())
- .Callable(opt ? "OrderedFlatMap" : "OrderedMap")
- .Add(0, input->HeadPtr())
- .Lambda(1)
- .Param("item")
- .Callable(input->Content())
- .Arg(0, "item")
- .Add(1, std::move(type))
- .Seal()
- .Seal()
- .Seal().Build();
-}
-
+template <bool Strong, class TSeqType>
+TExprNode::TPtr ExpandCastOverSequence(const TExprNode::TPtr& input, TExprContext& ctx) {
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for sequence";
+ const auto sourceType = input->Head().GetTypeAnn();
+ const auto targetType = input->GetTypeAnn();
+ const auto sourceItemType = sourceType->Cast<TSeqType>()->GetItemType();
+ const auto targetItemType = targetType->Cast<TSeqType>()->GetItemType();
+ const bool opt = CastMayFail<Strong>(sourceItemType, targetItemType);
+ auto type = ExpandType(input->Tail().Pos(), opt ? *ctx.MakeType<TOptionalExprType>(targetItemType) : *targetItemType, ctx);
+ return ctx.Builder(input->Pos())
+ .Callable(opt ? "OrderedFlatMap" : "OrderedMap")
+ .Add(0, input->HeadPtr())
+ .Lambda(1)
+ .Param("item")
+ .Callable(input->Content())
+ .Arg(0, "item")
+ .Add(1, std::move(type))
+ .Seal()
+ .Seal()
+ .Seal().Build();
+}
+
template <bool Strong>
TExprNode::TPtr ExpandCastOverDict(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Dict";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Dict";
const auto targetType = input->GetTypeAnn();
const auto targetDictType = targetType->Cast<TDictExprType>();
const TTypeAnnotationNode::TListType items = {targetDictType->GetKeyType(), targetDictType->GetPayloadType()};
@@ -866,7 +866,7 @@ TExprNode::TPtr ExpandCastOverDict(const TExprNode::TPtr& input, TExprContext& c
template <bool Strong>
TExprNode::TPtr ExpandCastOverTuple(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Tuple";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Tuple";
const auto targetType = input->GetTypeAnn();
const auto& targetItems = targetType->Cast<TTupleExprType>()->GetItems();
const auto sourceSize = input->Head().GetTypeAnn()->Cast<TTupleExprType>()->GetSize();
@@ -898,7 +898,7 @@ TExprNode::TPtr ExpandCastOverTuple(const TExprNode::TPtr& input, TExprContext&
template <bool Strong>
TExprNode::TPtr ExpandCastOverStruct(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Struct";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Struct";
const auto sourceType = input->Head().GetTypeAnn();
const auto targetType = input->GetTypeAnn();
const auto& sourceItems = sourceType->Cast<TStructExprType>()->GetItems();
@@ -938,7 +938,7 @@ TExprNode::TPtr ExpandCastOverStruct(const TExprNode::TPtr& input, TExprContext&
template <bool Strong>
TExprNode::TPtr ExpandCastOverVariant(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Variant";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Variant";
const auto targetType = input->GetTypeAnn();
const auto targetUnderType = targetType->Cast<TVariantExprType>()->GetUnderlyingType();
TExprNode::TListType variants, types;
@@ -991,7 +991,7 @@ TExprNode::TPtr ExpandCastOverVariant(const TExprNode::TPtr& input, TExprContext
template <bool Strong>
TExprNode::TPtr ExpandCastOverOptionalList(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for List?";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for List?";
const auto targetType = input->GetTypeAnn();
const auto targetItemType = targetType->Cast<TOptionalExprType>()->GetItemType()->Cast<TListExprType>()->GetItemType();
auto type = ExpandType(input->Tail().Pos(), *ctx.MakeType<TOptionalExprType>(targetItemType), ctx);
@@ -1039,7 +1039,7 @@ TExprNode::TPtr ExpandCastOverOptionalList(const TExprNode::TPtr& input, TExprCo
template <bool Strong>
TExprNode::TPtr ExpandCastOverOptionalDict(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Dict?";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Dict?";
const auto targetType = input->GetTypeAnn();
const auto targetDictType = targetType->Cast<TOptionalExprType>()->GetItemType()->Cast<TDictExprType>();
const TTypeAnnotationNode::TListType items = {targetDictType->GetKeyType(), targetDictType->GetPayloadType()};
@@ -1160,7 +1160,7 @@ TExprNode::TPtr FilterByOptionalItems(TExprNode::TPtr&& casted, TExprNode::TList
template <bool Strong>
TExprNode::TPtr ExpandCastOverOptionalTuple(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Tuple?";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Tuple?";
const auto sourceType = input->Head().GetTypeAnn();
const auto targetType = input->GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType();
const auto& sourceItems = sourceType->Cast<TTupleExprType>()->GetItems();
@@ -1257,7 +1257,7 @@ TExprNode::TPtr ExpandCastOverOptionalTuple(const TExprNode::TPtr& input, TExprC
template <bool Strong>
TExprNode::TPtr ExpandCastOverOptionalStruct(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Struct?";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Struct?";
const auto sourceType = input->Head().GetTypeAnn();
const auto targetType = input->GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType();
const auto& sourceItems = sourceType->Cast<TStructExprType>()->GetItems();
@@ -1371,7 +1371,7 @@ TExprNode::TPtr ExpandCastOverOptionalStruct(const TExprNode::TPtr& input, TExpr
template <bool Strong>
TExprNode::TPtr ExpandCastOverOptionalVariant(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Variant?";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " for Variant?";
const auto sourceType = input->Head().GetTypeAnn();
const auto targetType = input->GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType();
const auto sourceUnderType = sourceType->Cast<TVariantExprType>()->GetUnderlyingType();
@@ -1494,8 +1494,8 @@ TExprNode::TPtr ExpandCast(const TExprNode::TPtr& input, TExprContext& ctx) {
case ETypeAnnotationKind::Tuple: return ExpandCastOverTuple<Strong>(input, ctx);
case ETypeAnnotationKind::Struct: return ExpandCastOverStruct<Strong>(input, ctx);
case ETypeAnnotationKind::Variant: return ExpandCastOverVariant<Strong>(input, ctx);
- case ETypeAnnotationKind::Stream: return ExpandCastOverSequence<Strong, TStreamExprType>(input, ctx);
- case ETypeAnnotationKind::Flow: return ExpandCastOverSequence<Strong, TFlowExprType>(input, ctx);
+ case ETypeAnnotationKind::Stream: return ExpandCastOverSequence<Strong, TStreamExprType>(input, ctx);
+ case ETypeAnnotationKind::Flow: return ExpandCastOverSequence<Strong, TFlowExprType>(input, ctx);
default: break;
}
} else if (tKind == ETypeAnnotationKind::Optional) {
@@ -1513,7 +1513,7 @@ TExprNode::TPtr ExpandCast(const TExprNode::TPtr& input, TExprContext& ctx) {
default: break;
}
} else {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " as Map Just";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " as Map Just";
return ctx.Builder(input->Pos())
.Callable("Map")
.Callable(0, input->Content())
@@ -1529,7 +1529,7 @@ TExprNode::TPtr ExpandCast(const TExprNode::TPtr& input, TExprContext& ctx) {
.Seal().Build();
}
} else {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " as Just";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content() << " as Just";
return ctx.Builder(input->Pos())
.Callable("Just")
.Callable(0, input->Content())
@@ -1552,7 +1552,7 @@ TExprNode::TPtr ExpandAlterTo(const TExprNode::TPtr& node, TExprContext& ctx) {
const auto targetType = node->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (targetType->GetKind() == ETypeAnnotationKind::Null) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content() << " to " << *targetType;
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content() << " to " << *targetType;
return ctx.Builder(node->Pos())
.Callable("If")
.Callable(0, "Exists")
@@ -1568,7 +1568,7 @@ TExprNode::TPtr ExpandAlterTo(const TExprNode::TPtr& node, TExprContext& ctx) {
}
if (CastMayFail<true>(sourceType, targetType)) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
auto type = ExpandType(node->Child(1)->Pos(), *ctx.MakeType<TOptionalExprType>(targetType), ctx);
return ctx.Builder(node->Pos())
.Callable("IfPresent")
@@ -1587,7 +1587,7 @@ TExprNode::TPtr ExpandAlterTo(const TExprNode::TPtr& node, TExprContext& ctx) {
}
auto casted = ctx.NewCallable(node->Pos(), "StrictCast", {node->HeadPtr(), node->ChildPtr(1)});
- YQL_CLOG(DEBUG, CorePeepHole) << "Replace " << node->Content() << " on " << casted->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Replace " << node->Content() << " on " << casted->Content();
return ctx.ReplaceNode(node->Child(2)->TailPtr(), node->Child(2)->Head().Head(), std::move(casted));
}
@@ -1680,10 +1680,10 @@ TExprNode::TPtr ExpandSqlIn(const TExprNode::TPtr& input, TExprContext& ctx) {
if (collectionType->GetKind() == ETypeAnnotationKind::List)
{
if (collectionType->Cast<TListExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Struct) {
- YQL_CLOG(DEBUG, CorePeepHole) << "IN List of Structs";
+ YQL_CLOG(DEBUG, CorePeepHole) << "IN List of Structs";
dict = BuildDictOverListOfStructs(input->Pos(), collection, dictKeyType, ctx);
} else {
- YQL_CLOG(DEBUG, CorePeepHole) << "IN List";
+ YQL_CLOG(DEBUG, CorePeepHole) << "IN List";
dict = BuildDictOverList(input->Pos(), collection, ctx);
dictKeyType = collectionType->Cast<TListExprType>()->GetItemType();
}
@@ -1699,15 +1699,15 @@ TExprNode::TPtr ExpandSqlIn(const TExprNode::TPtr& input, TExprContext& ctx) {
.Seal()
.Build();
}
- YQL_CLOG(DEBUG, CorePeepHole) << "IN Tuple";
+ YQL_CLOG(DEBUG, CorePeepHole) << "IN Tuple";
dict = BuildDictOverTuple(std::move(collection), dictKeyType, ctx);
} else if (collectionType->GetKind() == ETypeAnnotationKind::EmptyDict) {
- YQL_CLOG(DEBUG, CorePeepHole) << "IN EmptyDict";
+ YQL_CLOG(DEBUG, CorePeepHole) << "IN EmptyDict";
} else if (collectionType->GetKind() == ETypeAnnotationKind::EmptyList) {
- YQL_CLOG(DEBUG, CorePeepHole) << "IN EmptyList";
+ YQL_CLOG(DEBUG, CorePeepHole) << "IN EmptyList";
} else {
YQL_ENSURE(collectionType->GetKind() == ETypeAnnotationKind::Dict);
- YQL_CLOG(DEBUG, CorePeepHole) << "IN Dict";
+ YQL_CLOG(DEBUG, CorePeepHole) << "IN Dict";
dict = collection;
dictKeyType = collectionType->Cast<TDictExprType>()->GetKeyType();
}
@@ -1717,20 +1717,20 @@ TExprNode::TPtr ExpandSqlIn(const TExprNode::TPtr& input, TExprContext& ctx) {
const auto justFalseNode = ctx.NewCallable(input->Pos(), "Just", { falseNode });
if (ansiIn && !dict) {
- YQL_CLOG(DEBUG, CorePeepHole) << "ANSI IN: with statically deduced empty collection";
+ YQL_CLOG(DEBUG, CorePeepHole) << "ANSI IN: with statically deduced empty collection";
return lookupType->HasOptionalOrNull() ? justFalseNode: falseNode;
}
TExprNode::TPtr contains = falseNode;
if (!dictKeyType) {
- YQL_CLOG(DEBUG, CorePeepHole) << "IN: Trivial Contains() due to statically deduced empty collection";
+ YQL_CLOG(DEBUG, CorePeepHole) << "IN: Trivial Contains() due to statically deduced empty collection";
} else if (NUdf::ECastOptions::Impossible & CastResult<true>(lookupType, dictKeyType)) {
- YQL_CLOG(DEBUG, CorePeepHole) << "IN: Trivial Contains() due to uncompatible type of lookup (" << *lookupType
+ YQL_CLOG(DEBUG, CorePeepHole) << "IN: Trivial Contains() due to uncompatible type of lookup (" << *lookupType
<< ") and collection item (" << *dictKeyType << ")";
} else {
YQL_ENSURE(dict);
contains = ctx.Builder(input->Pos())
- .Callable("Contains")
+ .Callable("Contains")
.Add(0, dict)
.Add(1, lookup)
.Seal()
@@ -1739,7 +1739,7 @@ TExprNode::TPtr ExpandSqlIn(const TExprNode::TPtr& input, TExprContext& ctx) {
const bool nullableCollectionItems = IsSqlInCollectionItemsNullable(NNodes::TCoSqlIn(input));
if (ansiIn && (nullableCollectionItems || lookupType->HasOptionalOrNull())) {
- YQL_CLOG(DEBUG, CorePeepHole) << "ANSI IN: with nullable items in collection or lookup";
+ YQL_CLOG(DEBUG, CorePeepHole) << "ANSI IN: with nullable items in collection or lookup";
YQL_ENSURE(dict);
const auto trueNode = MakeBool(input->Pos(), true, ctx);
const auto nullNode = MakeNull(input->Pos(), ctx);
@@ -1855,7 +1855,7 @@ TExprNode::TPtr CleckClosureOnUpperLambdaOverList(const TExprNode::TPtr& input,
const auto outerLambda = lambda->GetDependencyScope()->first;
if (outerLambda && lambda != outerLambda &&
!SkipCallables(input->Head(), SkippableCallables).IsCallable({"Collect", "AsList", "List", "ListIf"})) {
- YQL_CLOG(DEBUG, CorePeepHole) << input->Content() << " closure on upper lambda over list";
+ YQL_CLOG(DEBUG, CorePeepHole) << input->Content() << " closure on upper lambda over list";
return ctx.ChangeChild(*input, 0U, ctx.NewCallable(input->Head().Pos(), "Collect", {input->HeadPtr()}));
}
}
@@ -1867,7 +1867,7 @@ TExprNode::TPtr CleckClosureOnUpperLambdaOverList(const TExprNode::TPtr& input,
template <bool Inverse = false>
TExprNode::TPtr ExpandFilter(const TExprNode::TPtr& input, TExprContext& ctx) {
if (ETypeAnnotationKind::Optional == input->GetTypeAnn()->GetKind()) {
- YQL_CLOG(DEBUG, CorePeepHole) << input->Content() << " over Optional";
+ YQL_CLOG(DEBUG, CorePeepHole) << input->Content() << " over Optional";
auto none = ctx.Builder(input->Pos())
.Callable("Nothing")
.Add(0, ExpandType(input->Pos(), *input->GetTypeAnn(), ctx))
@@ -1926,7 +1926,7 @@ IGraphTransformer::TStatus PeepHoleCommonStage(const TExprNode::TPtr& input, TEx
}
IGraphTransformer::TStatus PeepHoleFinalStage(const TExprNode::TPtr& input, TExprNode::TPtr& output,
- TExprContext& ctx, TTypeAnnotationContext& types, bool* hasNonDeterministicFunctions,
+ TExprContext& ctx, TTypeAnnotationContext& types, bool* hasNonDeterministicFunctions,
const TPeepHoleOptimizerMap& optimizers, const TNonDeterministicOptimizerMap& nonDetOptimizers)
{
TOptimizeExprSettings settings(&types);
@@ -1934,9 +1934,9 @@ IGraphTransformer::TStatus PeepHoleFinalStage(const TExprNode::TPtr& input, TExp
return OptimizeExpr(input, output, [hasNonDeterministicFunctions, &types, &nonDetOptimizers, &optimizers](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
if (const auto nrule = nonDetOptimizers.find(node->Content()); nonDetOptimizers.cend() != nrule) {
- if (hasNonDeterministicFunctions) {
- *hasNonDeterministicFunctions = true;
- }
+ if (hasNonDeterministicFunctions) {
+ *hasNonDeterministicFunctions = true;
+ }
return (nrule->second)(node, ctx, types);
}
@@ -1946,18 +1946,18 @@ IGraphTransformer::TStatus PeepHoleFinalStage(const TExprNode::TPtr& input, TExp
}, ctx, settings);
}
-void AddStandardTransformers(TTransformationPipeline& pipelene, IGraphTransformer* typeAnnotator) {
+void AddStandardTransformers(TTransformationPipeline& pipelene, IGraphTransformer* typeAnnotator) {
auto issueCode = TIssuesIds::CORE_EXEC;
- pipelene.AddServiceTransformers(issueCode);
- if (typeAnnotator) {
- pipelene.Add(*typeAnnotator, "TypeAnnotation", issueCode);
- } else {
- pipelene.AddTypeAnnotationTransformer(issueCode);
- }
+ pipelene.AddServiceTransformers(issueCode);
+ if (typeAnnotator) {
+ pipelene.Add(*typeAnnotator, "TypeAnnotation", issueCode);
+ } else {
+ pipelene.AddTypeAnnotationTransformer(issueCode);
+ }
- pipelene.AddPostTypeAnnotation(true, issueCode);
- pipelene.Add(TExprLogTransformer::Sync("PeepHoleOpt", NLog::EComponent::CorePeepHole, NLog::ELevel::TRACE),
- "PeepHoleOptTrace", issueCode, "PeepHoleOptTrace");
+ pipelene.AddPostTypeAnnotation(true, issueCode);
+ pipelene.Add(TExprLogTransformer::Sync("PeepHoleOpt", NLog::EComponent::CorePeepHole, NLog::ELevel::TRACE),
+ "PeepHoleOptTrace", issueCode, "PeepHoleOptTrace");
}
template<bool FlatOrMulti>
@@ -1987,7 +1987,7 @@ TExprNode::TPtr ExpandFlatMap(const TExprNode::TPtr& node, TExprContext& ctx) {
if ((node->Head().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional || body.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) &&
body.IsCallable({"Just","AsList"}) && body.ChildrenSize() == 1U) {
- YQL_CLOG(DEBUG, CorePeepHole) << node->Content() << " to " << map;
+ YQL_CLOG(DEBUG, CorePeepHole) << node->Content() << " to " << map;
return ctx.Builder(node->Pos())
.Callable(map)
.Add(0, node->HeadPtr())
@@ -2028,7 +2028,7 @@ TExprNode::TPtr ExpandFlatMap(const TExprNode::TPtr& node, TExprContext& ctx) {
if (!haveSharedCallables) {
constexpr auto filter = Ordered ? "OrderedFilter" : "Filter";
- YQL_CLOG(DEBUG, CorePeepHole) << node->Content() << " over " << body.Content() << " to " << filter;
+ YQL_CLOG(DEBUG, CorePeepHole) << node->Content() << " over " << body.Content() << " to " << filter;
auto ret = ctx.Builder(node->Pos())
.Callable(filter)
.Add(0, node->HeadPtr())
@@ -2056,7 +2056,7 @@ TExprNode::TPtr ExpandFlatMap(const TExprNode::TPtr& node, TExprContext& ctx) {
}
if (ETypeAnnotationKind::Optional == node->GetTypeAnn()->GetKind()) {
- YQL_CLOG(DEBUG, CorePeepHole) << node->Content() << " over Optional";
+ YQL_CLOG(DEBUG, CorePeepHole) << node->Content() << " over Optional";
return ctx.Builder(node->Pos())
.Callable("IfPresent")
.Add(0, node->HeadPtr())
@@ -2095,13 +2095,13 @@ TExprNode::TPtr OptimizeMultiMap(const TExprNode::TPtr& node, TExprContext& ctx)
}
TExprNode::TPtr LikelyExclude(const TExprNode::TPtr& node, TExprContext&) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Exclude " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Exclude " << node->Content();
return node->HeadPtr();
}
TExprNode::TPtr WrapIteratorForOptionalList(const TExprNode::TPtr& node, TExprContext& ctx) {
if (node->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Wrap " << node->Content() << " for optional list.";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Wrap " << node->Content() << " for optional list.";
return ctx.Builder(node->Pos())
.Callable("IfPresent")
.Add(0, node->HeadPtr())
@@ -2128,7 +2128,7 @@ TExprNode::TPtr WrapIteratorForOptionalList(const TExprNode::TPtr& node, TExprCo
TExprNode::TPtr MapForOptionalContainer(const TExprNode::TPtr& node, TExprContext& ctx) {
if (node->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Map " << node->Content() << " for optional container.";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Map " << node->Content() << " for optional container.";
return ctx.Builder(node->Pos())
.Callable("Map")
.Add(0, node->HeadPtr())
@@ -2147,14 +2147,14 @@ TExprNode::TPtr MapForOptionalContainer(const TExprNode::TPtr& node, TExprContex
template <bool BoolResult, bool ByList = false>
TExprNode::TPtr RewriteSearchByKeyForTypesMismatch(const TExprNode::TPtr& node, TExprContext& ctx) {
const auto type = node->Head().GetTypeAnn();
- if constexpr (BoolResult) {
- if (IsDataOrOptionalOfData(type)) {
- return node;
- }
- }
-
+ if constexpr (BoolResult) {
+ if (IsDataOrOptionalOfData(type)) {
+ return node;
+ }
+ }
+
if (type->GetKind() == ETypeAnnotationKind::Optional) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Wrap " << node->Content() << " for optional collection.";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Wrap " << node->Content() << " for optional collection.";
const auto unwrapType = type->Cast<TOptionalExprType>()->GetItemType();
const auto payloadType = ByList ? unwrapType->Cast<TListExprType>()->GetItemType() : unwrapType->Cast<TDictExprType>()->GetPayloadType();
return ctx.Builder(node->Pos())
@@ -2174,7 +2174,7 @@ TExprNode::TPtr RewriteSearchByKeyForTypesMismatch(const TExprNode::TPtr& node,
const auto keyType = ByList ? type->Cast<TListExprType>()->GetItemType() : type->Cast<TDictExprType>()->GetKeyType();
if (!IsSameAnnotation(*keyType, *node->Tail().GetTypeAnn())) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Wrap " << node->Content() << " for key type mismatch.";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Wrap " << node->Content() << " for key type mismatch.";
const auto payloadType = ByList ? type->Cast<TListExprType>()->GetItemType() : type->Cast<TDictExprType>()->GetPayloadType();
return ctx.Builder(node->Pos())
.Callable("AlterTo")
@@ -2198,7 +2198,7 @@ TExprNode::TPtr ExpandListHas(const TExprNode::TPtr& input, TExprContext& ctx) {
const auto type = input->Head().GetTypeAnn();
if (ETypeAnnotationKind::List == type->GetKind() &&
IsSameAnnotation(*type->Cast<TListExprType>()->GetItemType(), *input->Tail().GetTypeAnn())) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content();
return ctx.Builder(input->Pos())
.Callable("Exists")
.Callable(0, "Head")
@@ -2221,14 +2221,14 @@ TExprNode::TPtr ExpandListHas(const TExprNode::TPtr& input, TExprContext& ctx) {
template <bool Flat, bool List>
TExprNode::TPtr ExpandContainerIf(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << input->Content();
auto item = Flat ? input->TailPtr() : ctx.NewCallable(input->Tail().Pos(), List ? "AsList" : "Just", {input->TailPtr()});
auto none = ctx.NewCallable(input->Tail().Pos(), List ? "List" : "Nothing", {ExpandType(input->Tail().Pos(), *input->GetTypeAnn(), ctx)});
return ctx.NewCallable(input->Pos(), "If", {input->HeadPtr(), std::move(item), std::move(none)});
}
TExprNode::TPtr ExpandPartitionsByKeys(const TExprNode::TPtr& node, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
TExprNode::TPtr sort;
if (node->Child(2)->IsCallable("Void") || node->Child(3)->IsCallable("Void")) {
sort = ctx.NewCallable(node->Pos(), "Sort", {node->HeadPtr(), MakeBool<true>(node->Pos(), ctx), node->ChildPtr(1)});
@@ -2307,7 +2307,7 @@ TExprNode::TPtr ExpandPartitionsByKeys(const TExprNode::TPtr& node, TExprContext
}
TExprNode::TPtr ExpandIsKeySwitch(const TExprNode::TPtr& node, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
return ctx.Builder(node->Pos())
.Callable("AggrNotEquals")
.Add(0, ctx.ReplaceNode(node->Child(2)->TailPtr(), node->Child(2)->Head().Head(), node->ChildPtr(0)))
@@ -2316,7 +2316,7 @@ TExprNode::TPtr ExpandIsKeySwitch(const TExprNode::TPtr& node, TExprContext& ctx
}
TExprNode::TPtr ExpandMux(const TExprNode::TPtr& node, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
const auto varType = ExpandType(node->Pos(), *GetSeqItemType(node->GetTypeAnn()), ctx);
TExprNode::TListType lists;
switch (node->Head().GetTypeAnn()->GetKind()) {
@@ -2325,7 +2325,7 @@ TExprNode::TPtr ExpandMux(const TExprNode::TPtr& node, TExprContext& ctx) {
lists.resize(type->GetSize());
for (ui32 i = 0U; i < lists.size(); ++i) {
lists[i] = ctx.Builder(node->Head().Pos())
- .Callable("OrderedMap")
+ .Callable("OrderedMap")
.Callable(0, "Nth")
.Add(0, node->HeadPtr())
.Atom(1, ToString(i), TNodeFlags::Default)
@@ -2348,7 +2348,7 @@ TExprNode::TPtr ExpandMux(const TExprNode::TPtr& node, TExprContext& ctx) {
for (const auto& item : type->GetItems()) {
lists.emplace_back(
ctx.Builder(node->Head().Pos())
- .Callable("OrderedMap")
+ .Callable("OrderedMap")
.Callable(0, "Member")
.Add(0, node->HeadPtr())
.Atom(1, item->GetName())
@@ -2373,7 +2373,7 @@ TExprNode::TPtr ExpandMux(const TExprNode::TPtr& node, TExprContext& ctx) {
}
TExprNode::TPtr ExpandLMap(const TExprNode::TPtr& node, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
return ctx.Builder(node->Pos())
.Callable("Collect")
.Apply(0, node->Tail())
@@ -2387,7 +2387,7 @@ TExprNode::TPtr ExpandLMap(const TExprNode::TPtr& node, TExprContext& ctx) {
}
TExprNode::TPtr ExpandDemux(const TExprNode::TPtr& node, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
TExprNode::TListType lists;
switch (node->GetTypeAnn()->GetKind()) {
case ETypeAnnotationKind::Tuple: {
@@ -2395,7 +2395,7 @@ TExprNode::TPtr ExpandDemux(const TExprNode::TPtr& node, TExprContext& ctx) {
lists.resize(type->GetSize());
for (ui32 i = 0U; i < lists.size(); ++i) {
lists[i] = ctx.Builder(node->Head().Pos())
- .Callable("OrderedFlatMap")
+ .Callable("OrderedFlatMap")
.Add(0, node->HeadPtr())
.Lambda(1)
.Param("item")
@@ -2416,7 +2416,7 @@ TExprNode::TPtr ExpandDemux(const TExprNode::TPtr& node, TExprContext& ctx) {
ctx.Builder(node->Head().Pos())
.List()
.Atom(0, item->GetName())
- .Callable(1, "OrderedFlatMap")
+ .Callable(1, "OrderedFlatMap")
.Add(0, node->HeadPtr())
.Lambda(1)
.Param("item")
@@ -2438,7 +2438,7 @@ TExprNode::TPtr ExpandDemux(const TExprNode::TPtr& node, TExprContext& ctx) {
}
TExprNode::TPtr ExpandOptionalReduce(const TExprNode::TPtr& node, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
return ctx.Builder(node->Pos())
.Callable("IfPresent")
.Add(0, node->ChildPtr(0))
@@ -2469,7 +2469,7 @@ TExprNode::TPtr ExpandAggrMinMax(const TExprNode::TPtr& node, TExprContext& ctx)
}
if (ETypeAnnotationKind::Optional == node->GetTypeAnn()->GetKind()) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content() << " over Optional";
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content() << " over Optional";
return ctx.Builder(node->Pos())
.Callable("OptionalReduce")
.Add(0, node->HeadPtr())
@@ -2485,7 +2485,7 @@ TExprNode::TPtr ExpandAggrMinMax(const TExprNode::TPtr& node, TExprContext& ctx)
.Seal().Build();
}
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
return ctx.Builder(node->Pos())
.Callable("If")
.Callable(0, MinOrMax ? "AggrLessOrEqual" : "AggrGreaterOrEqual")
@@ -2534,27 +2534,27 @@ TExprNode::TPtr OptimizeMap(const TExprNode::TPtr& node, TExprContext& ctx) {
}
if (node->Head().IsCallable("NarrowMap")) {
- if (!arg.IsUsedInDependsOn()) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Fuse " << node->Content() << " with " << node->Head().Content();
- const auto width = node->Head().Tail().Head().ChildrenSize();
- auto lambda = ctx.Builder(node->Pos())
- .Lambda()
- .Params("items", width)
- .Apply(node->Tail())
- .With(0)
- .Apply(node->Head().Tail())
- .With("items")
- .Seal()
- .Done()
- .Seal()
- .Seal().Build();
- return ctx.ChangeChild(node->Head(), 1U, std::move(lambda));
- }
+ if (!arg.IsUsedInDependsOn()) {
+ YQL_CLOG(DEBUG, CorePeepHole) << "Fuse " << node->Content() << " with " << node->Head().Content();
+ const auto width = node->Head().Tail().Head().ChildrenSize();
+ auto lambda = ctx.Builder(node->Pos())
+ .Lambda()
+ .Params("items", width)
+ .Apply(node->Tail())
+ .With(0)
+ .Apply(node->Head().Tail())
+ .With("items")
+ .Seal()
+ .Done()
+ .Seal()
+ .Seal().Build();
+ return ctx.ChangeChild(node->Head(), 1U, std::move(lambda));
+ }
}
if (1U == node->Head().UseCount() && !arg.IsUsedInDependsOn()) {
if (node->Head().IsCallable({"Map", "OrderedMap"})) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Fuse " << node->Content() << " over " << node->Head().Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Fuse " << node->Content() << " over " << node->Head().Content();
auto lambda = ctx.Builder(node->Pos())
.Lambda()
.Param("it")
@@ -2609,7 +2609,7 @@ TExprNode::TPtr OptimizeTake(const TExprNode::TPtr& node, TExprContext& ctx) {
}
TExprNode::TPtr ExpandAsSet(const TExprNode::TPtr& node, TExprContext& ctx) {
- YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
+ YQL_CLOG(DEBUG, CorePeepHole) << "Expand " << node->Content();
return ctx.Builder(node->Pos())
.Callable("AsDict")
.Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
@@ -5698,7 +5698,7 @@ struct TPeepHoleRules {
{"AlterTo", &ExpandAlterTo},
{"SqlIn", &ExpandSqlIn},
{"Lookup", &RewriteSearchByKeyForTypesMismatch<false>},
- {"Contains", &RewriteSearchByKeyForTypesMismatch<true>},
+ {"Contains", &RewriteSearchByKeyForTypesMismatch<true>},
{"ListHas", &ExpandListHas},
{"Map", &CleckClosureOnUpperLambdaOverList},
{"OrderedMap", &CleckClosureOnUpperLambdaOverList},
@@ -5781,7 +5781,7 @@ struct TPeepHoleRules {
{"AddMember", &ExpandAddMember},
{"ReplaceMember", &ExpandReplaceMember},
{"RemoveMember", &ExpandRemoveMember},
- {"RemovePrefixMembers", &ExpandRemovePrefixMembers},
+ {"RemovePrefixMembers", &ExpandRemovePrefixMembers},
{"AsSet", &ExpandAsSet},
{"ForceRemoveMember", &ExpandRemoveMember},
{"DivePrefixMembers", &ExpandDivePrefixMembers},
@@ -5851,12 +5851,12 @@ THolder<IGraphTransformer> CreatePeepHoleCommonStageTransformer(TTypeAnnotationC
TTransformationPipeline pipeline(&types);
if (peepholeSettings.CommonConfig) {
peepholeSettings.CommonConfig->AfterCreate(&pipeline);
- }
+ }
AddStandardTransformers(pipeline, typeAnnotator);
if (peepholeSettings.CommonConfig) {
peepholeSettings.CommonConfig->AfterTypeAnnotation(&pipeline);
- }
+ }
auto issueCode = TIssuesIds::CORE_EXEC;
@@ -5871,8 +5871,8 @@ THolder<IGraphTransformer> CreatePeepHoleCommonStageTransformer(TTypeAnnotationC
if (peepholeSettings.CommonConfig) {
peepholeSettings.CommonConfig->AfterOptimize(&pipeline);
- }
-
+ }
+
return pipeline.BuildWithNoArgChecks(false);
}
@@ -5885,12 +5885,12 @@ THolder<IGraphTransformer> CreatePeepHoleFinalStageTransformer(TTypeAnnotationCo
TTransformationPipeline pipeline(&types);
if (peepholeSettings.FinalConfig) {
peepholeSettings.FinalConfig->AfterCreate(&pipeline);
- }
+ }
AddStandardTransformers(pipeline, typeAnnotator);
if (peepholeSettings.FinalConfig) {
peepholeSettings.FinalConfig->AfterTypeAnnotation(&pipeline);
- }
+ }
auto issueCode = TIssuesIds::CORE_EXEC;
@@ -5915,15 +5915,15 @@ THolder<IGraphTransformer> CreatePeepHoleFinalStageTransformer(TTypeAnnotationCo
if (peepholeSettings.FinalConfig) {
peepholeSettings.FinalConfig->AfterOptimize(&pipeline);
- }
-
+ }
+
return pipeline.BuildWithNoArgChecks(false);
}
}
-IGraphTransformer::TStatus DoPeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx,
- IGraphTransformer& commonTransformer, IGraphTransformer& finalTransformer)
+IGraphTransformer::TStatus DoPeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx,
+ IGraphTransformer& commonTransformer, IGraphTransformer& finalTransformer)
{
output = input;
@@ -5944,17 +5944,17 @@ IGraphTransformer::TStatus DoPeepHoleOptimizeNode(const TExprNode::TPtr& input,
}
template <bool EnableNewOptimizers>
-IGraphTransformer::TStatus PeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output,
+IGraphTransformer::TStatus PeepHoleOptimizeNode(const TExprNode::TPtr& input, TExprNode::TPtr& output,
TExprContext& ctx, TTypeAnnotationContext& types, IGraphTransformer* typeAnnotator,
bool& hasNonDeterministicFunctions, const TPeepholeSettings& peepholeSettings)
-{
- hasNonDeterministicFunctions = false;
+{
+ hasNonDeterministicFunctions = false;
const auto commonTransformer = CreatePeepHoleCommonStageTransformer<EnableNewOptimizers>(types, typeAnnotator, peepholeSettings);
const auto finalTransformer = CreatePeepHoleFinalStageTransformer<EnableNewOptimizers>(types, typeAnnotator,
&hasNonDeterministicFunctions, peepholeSettings);
- return DoPeepHoleOptimizeNode(input, output, ctx, *commonTransformer, *finalTransformer);
-}
-
+ return DoPeepHoleOptimizeNode(input, output, ctx, *commonTransformer, *finalTransformer);
+}
+
THolder<IGraphTransformer> MakePeepholeOptimization(TTypeAnnotationContextPtr typeAnnotationContext, const IPipelineConfigurator* config) {
TPeepholeSettings peepholeSettings;
peepholeSettings.CommonConfig = peepholeSettings.FinalConfig = config;
diff --git a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h
index a357c5b395..afcfcf71e0 100644
--- a/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h
+++ b/ydb/library/yql/core/peephole_opt/yql_opt_peephole_physical.h
@@ -5,8 +5,8 @@
namespace NYql {
-struct IPipelineConfigurator;
-
+struct IPipelineConfigurator;
+
struct TPeepholeSettings {
const IPipelineConfigurator* CommonConfig = nullptr;
const IPipelineConfigurator* FinalConfig = nullptr;
diff --git a/ydb/library/yql/core/services/mounts/yql_mounts.cpp b/ydb/library/yql/core/services/mounts/yql_mounts.cpp
index 7d0e221799..2361dbf342 100644
--- a/ydb/library/yql/core/services/mounts/yql_mounts.cpp
+++ b/ydb/library/yql/core/services/mounts/yql_mounts.cpp
@@ -115,10 +115,10 @@ namespace NYql {
IModuleResolver::TPtr& moduleResolver,
const TVector<NUserData::TUserData>& userData,
const THashMap<TString, TString>& clusterMapping,
- const THashSet<TString>& sqlFlags,
+ const THashSet<TString>& sqlFlags,
bool optimizeLibraries,
THolder<TExprContext> ownedCtx)
- {
+ {
auto ctx = rawCtx ? rawCtx : ownedCtx.Get();
Y_ENSURE(ctx);
TUserDataTable mounts = GetYqlDefaultMounts();
@@ -150,7 +150,7 @@ namespace NYql {
bool GetYqlDefaultModuleResolver(
TExprContext& ctx,
IModuleResolver::TPtr& moduleResolver,
- const THashMap<TString, TString>& clusterMapping,
+ const THashMap<TString, TString>& clusterMapping,
bool optimizeLibraries) {
return !GetYqlModuleResolverImpl(&ctx, moduleResolver, {}, clusterMapping, {}, optimizeLibraries, nullptr).empty();
}
diff --git a/ydb/library/yql/core/services/mounts/yql_mounts.h b/ydb/library/yql/core/services/mounts/yql_mounts.h
index 27b30c4ccd..a61a8a4aa2 100644
--- a/ydb/library/yql/core/services/mounts/yql_mounts.h
+++ b/ydb/library/yql/core/services/mounts/yql_mounts.h
@@ -10,7 +10,7 @@ TUserDataTable GetYqlDefaultMounts();
bool GetYqlDefaultModuleResolver(
TExprContext& ctx,
IModuleResolver::TPtr& moduleResolver,
- const THashMap<TString, TString>& clusterMapping = {},
+ const THashMap<TString, TString>& clusterMapping = {},
bool optimizeLibraries = true);
bool GetYqlDefaultModuleResolverWithContext(
@@ -23,7 +23,7 @@ TUserDataTable GetYqlModuleResolver(
IModuleResolver::TPtr& moduleResolver,
const TVector<NUserData::TUserData>& userData,
const THashMap<TString, TString>& clusterMapping,
- const THashSet<TString>& sqlFlags,
+ const THashSet<TString>& sqlFlags,
bool optimizeLibraries = true);
} // namespace NYql
diff --git a/ydb/library/yql/core/services/yql_eval_expr.cpp b/ydb/library/yql/core/services/yql_eval_expr.cpp
index 811f22a605..07d41d47e2 100644
--- a/ydb/library/yql/core/services/yql_eval_expr.cpp
+++ b/ydb/library/yql/core/services/yql_eval_expr.cpp
@@ -1,7 +1,7 @@
#include "yql_eval_expr.h"
#include "yql_transform_pipeline.h"
-#include "yql_out_transformers.h"
-
+#include "yql_out_transformers.h"
+
#include <ydb/library/yql/ast/serialize/yql_expr_serialize.h>
#include <ydb/library/yql/core/type_ann/type_ann_core.h>
#include <ydb/library/yql/core/type_ann/type_ann_expr.h>
@@ -12,9 +12,9 @@
#include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h>
#include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h>
#include <ydb/library/yql/utils/log/log.h>
-
+
#include <library/cpp/yson/node/node_io.h>
-#include <library/cpp/string_utils/base64/base64.h>
+#include <library/cpp/string_utils/base64/base64.h>
#include <util/string/builder.h>
@@ -83,86 +83,86 @@ bool CheckPendingArgs(const TExprNode& root, TNodeSet& visited, TNodeMap<const T
return true;
}
-class TMarkReachable {
-public:
- TNodeSet Reachable;
- TNodeMap<ui32> ExternalWorlds;
- TDeque<TExprNode::TPtr> ExternalWorldsList;
- bool HasConfigPending = false;
-
-public:
- void Scan(const TExprNode& node) {
- VisitExprByFirst(node, [this](const TExprNode& n) {
- if (n.IsCallable(ConfigureName)) {
- if (n.ChildrenSize() > 3 && n.Child(1)->Child(0)->Content() == ConfigProviderName) {
- bool pending = false;
- for (size_t i = 3; i < n.ChildrenSize(); ++i) {
- if (n.Child(i)->IsCallable("EvaluateAtom")) {
- pending = true;
- break;
- }
- }
- if (pending) {
- const TStringBuf command = n.Child(2)->Content();
- if (command == "AddFileByUrl") {
- PendingFileAliases.insert(n.Child(3)->Content());
- } else if (command == "AddFolderByUrl") {
- PendingFolderPrefixes.insert(n.Child(3)->Content());
- }
- }
- }
- }
- return true;
- });
- ScanImpl(node);
+class TMarkReachable {
+public:
+ TNodeSet Reachable;
+ TNodeMap<ui32> ExternalWorlds;
+ TDeque<TExprNode::TPtr> ExternalWorldsList;
+ bool HasConfigPending = false;
+
+public:
+ void Scan(const TExprNode& node) {
+ VisitExprByFirst(node, [this](const TExprNode& n) {
+ if (n.IsCallable(ConfigureName)) {
+ if (n.ChildrenSize() > 3 && n.Child(1)->Child(0)->Content() == ConfigProviderName) {
+ bool pending = false;
+ for (size_t i = 3; i < n.ChildrenSize(); ++i) {
+ if (n.Child(i)->IsCallable("EvaluateAtom")) {
+ pending = true;
+ break;
+ }
+ }
+ if (pending) {
+ const TStringBuf command = n.Child(2)->Content();
+ if (command == "AddFileByUrl") {
+ PendingFileAliases.insert(n.Child(3)->Content());
+ } else if (command == "AddFolderByUrl") {
+ PendingFolderPrefixes.insert(n.Child(3)->Content());
+ }
+ }
+ }
+ }
+ return true;
+ });
+ ScanImpl(node);
}
-private:
- void ScanImpl(const TExprNode& node) {
- if (!Visited.emplace(&node).second) {
- return;
- }
-
- if (node.IsCallable("Seq!")) {
- for (ui32 i = 1; i < node.ChildrenSize(); ++i) {
- auto lambda = node.Child(i);
- auto arg = lambda->Child(0)->ChildPtr(0);
- ui32 id = ExternalWorlds.size();
- YQL_ENSURE(ExternalWorlds.emplace(arg.Get(), id).second);
- ExternalWorldsList.push_back(arg);
- }
- }
-
- static THashSet<TStringBuf> FILE_CALLABLES = {"FilePath", "FileContent", "FolderPath"};
- if (node.IsCallable(FILE_CALLABLES)) {
- const auto alias = node.Head().Content();
- if (PendingFileAliases.contains(alias) || AnyOf(PendingFolderPrefixes, [alias](const TStringBuf prefix) { return alias.StartsWith(prefix); })) {
- for (auto& curr: CurrentEvalNodes) {
- Reachable.erase(curr);
- }
- HasConfigPending = true;
- }
+private:
+ void ScanImpl(const TExprNode& node) {
+ if (!Visited.emplace(&node).second) {
+ return;
}
- if (node.IsCallable("QuoteCode")) {
- Reachable.insert(&node);
- return;
+ if (node.IsCallable("Seq!")) {
+ for (ui32 i = 1; i < node.ChildrenSize(); ++i) {
+ auto lambda = node.Child(i);
+ auto arg = lambda->Child(0)->ChildPtr(0);
+ ui32 id = ExternalWorlds.size();
+ YQL_ENSURE(ExternalWorlds.emplace(arg.Get(), id).second);
+ ExternalWorldsList.push_back(arg);
+ }
+ }
+
+ static THashSet<TStringBuf> FILE_CALLABLES = {"FilePath", "FileContent", "FolderPath"};
+ if (node.IsCallable(FILE_CALLABLES)) {
+ const auto alias = node.Head().Content();
+ if (PendingFileAliases.contains(alias) || AnyOf(PendingFolderPrefixes, [alias](const TStringBuf prefix) { return alias.StartsWith(prefix); })) {
+ for (auto& curr: CurrentEvalNodes) {
+ Reachable.erase(curr);
+ }
+ HasConfigPending = true;
+ }
+ }
+
+ if (node.IsCallable("QuoteCode")) {
+ Reachable.insert(&node);
+ return;
}
-
- bool pop = false;
+
+ bool pop = false;
if (node.IsCallable(EvaluationFuncs) || node.IsCallable(SubqueryExpandFuncs)) {
- Reachable.insert(&node);
- CurrentEvalNodes.insert(&node);
- pop = true;
+ Reachable.insert(&node);
+ CurrentEvalNodes.insert(&node);
+ pop = true;
}
-
+
if (node.IsCallable({ "EvaluateIf!", "EvaluateFor!" })) {
- // scan predicate/list only
- if (node.ChildrenSize() > 1) {
- CurrentEvalNodes.insert(&node);
- pop = true;
- ScanImpl(*node.Child(1));
- }
+ // scan predicate/list only
+ if (node.ChildrenSize() > 1) {
+ CurrentEvalNodes.insert(&node);
+ pop = true;
+ ScanImpl(*node.Child(1));
+ }
} else if (node.IsCallable(SubqueryExpandFuncs)) {
// scan list only if it's wrapped by evaluation func
ui32 index = 0;
@@ -180,24 +180,24 @@ private:
}
}
}
- } else {
- for (const auto& child : node.Children()) {
- ScanImpl(*child);
- }
- }
- if (pop) {
- CurrentEvalNodes.erase(&node);
- }
+ } else {
+ for (const auto& child : node.Children()) {
+ ScanImpl(*child);
+ }
+ }
+ if (pop) {
+ CurrentEvalNodes.erase(&node);
+ }
}
-private:
- TNodeSet Visited;
- THashSet<TStringBuf> PendingFileAliases;
- THashSet<TStringBuf> PendingFolderPrefixes;
- TNodeSet CurrentEvalNodes;
-};
-
-
+private:
+ TNodeSet Visited;
+ THashSet<TStringBuf> PendingFileAliases;
+ THashSet<TStringBuf> PendingFolderPrefixes;
+ TNodeSet CurrentEvalNodes;
+};
+
+
bool ValidateCalcWorlds(const TExprNode& node, const TTypeAnnotationContext& types, TNodeSet& visited) {
if (!visited.emplace(&node).second) {
return true;
@@ -222,7 +222,7 @@ bool ValidateCalcWorlds(const TExprNode& node, const TTypeAnnotationContext& typ
}
for (auto& dataProvider : types.DataSources) {
- if (dataProvider->CanEvaluate(node)) {
+ if (dataProvider->CanEvaluate(node)) {
return true;
}
}
@@ -357,7 +357,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
if (ctx.Step.IsDone(TExprStep::ExprEval))
return IGraphTransformer::TStatus::Ok;
- YQL_CLOG(DEBUG, CoreEval) << "EvaluateExpression - start";
+ YQL_CLOG(DEBUG, CoreEval) << "EvaluateExpression - start";
bool pure = false;
TString nextProvider;
TMaybe<IDataProvider*> calcProvider;
@@ -368,14 +368,14 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
bool isCodePipeline = false;
TTransformationPipeline pipeline(&types);
pipeline.AddServiceTransformers();
- pipeline.AddPreTypeAnnotation();
+ pipeline.AddPreTypeAnnotation();
pipeline.AddExpressionEvaluation(functionRegistry);
- pipeline.AddIOAnnotation();
+ pipeline.AddIOAnnotation();
pipeline.AddTypeAnnotationTransformer();
pipeline.Add(CreateFunctorTransformer(
[&](TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) -> IGraphTransformer::TStatus {
output = input;
- if (!input->GetTypeAnn()) {
+ if (!input->GetTypeAnn()) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Lambda is not allowed as argument for function: " << input->Content()));
return IGraphTransformer::TStatus::Error;
}
@@ -407,8 +407,8 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
}), "TopLevelType", EYqlIssueCode::TIssuesIds_EIssueCode_DEFAULT_ERROR, "Ensure type of expression is correct");
const bool forSubGraph = true;
pipeline.AddPostTypeAnnotation(forSubGraph);
- pipeline.Add(TExprLogTransformer::Sync("EvalExpressionOpt", NLog::EComponent::CoreEval, NLog::ELevel::TRACE),
- "EvalOptTrace", EYqlIssueCode::TIssuesIds_EIssueCode_DEFAULT_ERROR, "EvalOptTrace");
+ pipeline.Add(TExprLogTransformer::Sync("EvalExpressionOpt", NLog::EComponent::CoreEval, NLog::ELevel::TRACE),
+ "EvalOptTrace", EYqlIssueCode::TIssuesIds_EIssueCode_DEFAULT_ERROR, "EvalOptTrace");
pipeline.AddOptimization(false);
pipeline.Add(CreateFunctorTransformer(
[&](TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) -> IGraphTransformer::TStatus {
@@ -431,14 +431,14 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
} else if (!calcTransfomer) {
for (auto& p : types.DataSources) {
TSyncMap syncList;
- if (p->CanBuildResult(*input, syncList)) {
+ if (p->CanBuildResult(*input, syncList)) {
bool canExec = true;
for (auto& x : syncList) {
if (x.first->Type() == TExprNode::World) {
continue;
}
- if (!p->GetExecWorld(*x.first, calcWorldRoot)) {
+ if (!p->GetExecWorld(*x.first, calcWorldRoot)) {
canExec = false;
break;
}
@@ -455,7 +455,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
}
if (canExec) {
- calcProvider = p.Get();
+ calcProvider = p.Get();
output = (*calcProvider.Get())->CleanupWorld(input, ctx);
if (!output) {
return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error);
@@ -474,7 +474,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
if (!calcWorldRoot) {
calcWorldRoot = ctx.NewWorld(input->Pos());
- calcWorldRoot->SetTypeAnn(ctx.MakeType<TUnitExprType>());
+ calcWorldRoot->SetTypeAnn(ctx.MakeType<TUnitExprType>());
calcWorldRoot->SetState(TExprNode::EState::ConstrComplete);
}
}
@@ -486,18 +486,18 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
auto fullTransformer = pipeline.Build();
- TMarkReachable marked;
- marked.Scan(*output);
-
- THashSet<TStringBuf> modifyCallables;
- modifyCallables.insert(WriteName);
- modifyCallables.insert(ConfigureName);
- modifyCallables.insert(CommitName);
- modifyCallables.insert("CommitAll!");
- for (auto& dataSink: types.DataSinks) {
- dataSink->FillModifyCallables(modifyCallables);
- }
-
+ TMarkReachable marked;
+ marked.Scan(*output);
+
+ THashSet<TStringBuf> modifyCallables;
+ modifyCallables.insert(WriteName);
+ modifyCallables.insert(ConfigureName);
+ modifyCallables.insert(CommitName);
+ modifyCallables.insert("CommitAll!");
+ for (auto& dataSink: types.DataSinks) {
+ dataSink->FillModifyCallables(modifyCallables);
+ }
+
IGraphTransformer::TStatus hasPendingEvaluations = IGraphTransformer::TStatus::Ok;
TOptimizeExprSettings settings(nullptr);
settings.VisitChanges = true;
@@ -845,7 +845,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
}
if (node->IsCallable("QuoteCode")) {
- if (marked.Reachable.find(node.Get()) == marked.Reachable.cend()) {
+ if (marked.Reachable.find(node.Get()) == marked.Reachable.cend()) {
hasPendingEvaluations = hasPendingEvaluations.Combine(IGraphTransformer::TStatus::Repeat);
return node;
}
@@ -856,7 +856,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
TNodeOnNodeOwnedMap knownArgs;
TNodeOnNodeOwnedMap visited;
- return QuoteCode(node->ChildPtr(0), ctx, knownArgs, visited, marked.ExternalWorlds);
+ return QuoteCode(node->ChildPtr(0), ctx, knownArgs, visited, marked.ExternalWorlds);
}
if (!node->IsCallable(EvaluationFuncs)) {
@@ -867,11 +867,11 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
return nullptr;
}
- if (marked.Reachable.find(node.Get()) == marked.Reachable.cend()) {
- if (marked.HasConfigPending) {
- ctx.Step.Repeat(TExprStep::Configure);
- }
- hasPendingEvaluations = hasPendingEvaluations.Combine(IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, marked.HasConfigPending));
+ if (marked.Reachable.find(node.Get()) == marked.Reachable.cend()) {
+ if (marked.HasConfigPending) {
+ ctx.Step.Repeat(TExprStep::Configure);
+ }
+ hasPendingEvaluations = hasPendingEvaluations.Combine(IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, marked.HasConfigPending));
return node;
}
@@ -880,18 +880,18 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
TNodeSet visited;
TNodeMap<const TExprNode*> activeArgs;
bool hasUnresolvedTypes = false;
- if (!CheckPendingArgs(*newArg, visited, activeArgs, marked.ExternalWorlds, ctx, false, hasUnresolvedTypes)) {
+ if (!CheckPendingArgs(*newArg, visited, activeArgs, marked.ExternalWorlds, ctx, false, hasUnresolvedTypes)) {
return nullptr;
}
if (hasUnresolvedTypes) {
- YQL_CLOG(DEBUG, CoreEval) << "EvaluateExpression - has unresolved types";
+ YQL_CLOG(DEBUG, CoreEval) << "EvaluateExpression - has unresolved types";
return node;
}
}
TNodeOnNodeOwnedMap externalWorldReplaces;
- for (auto& x : marked.ExternalWorlds) {
+ for (auto& x : marked.ExternalWorlds) {
externalWorldReplaces.emplace(x.first, ctx.NewWorld(x.first->Pos()));
}
@@ -907,7 +907,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
settings.VisitChanges = true;
auto status = OptimizeExpr(clonedArg, clonedArg, [&](const TExprNode::TPtr& node, TExprContext& ctx) {
Y_UNUSED(ctx);
- if (node->IsCallable(modifyCallables) && node->ChildrenSize() > 0) {
+ if (node->IsCallable(modifyCallables) && node->ChildrenSize() > 0) {
return node->ChildPtr(0);
}
@@ -1009,23 +1009,23 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
return nullptr;
}
- dataNode = dataNode.AsList().front();
- }
- TString value;
- if (dataNode.IsString()) {
- value = dataNode.AsString();
+ dataNode = dataNode.AsList().front();
+ }
+ TString value;
+ if (dataNode.IsString()) {
+ value = dataNode.AsString();
} else {
- YQL_ENSURE(dataNode.IsList() && dataNode.AsList().size() == 1 && dataNode.AsList().front().IsString(), "Unexpected atom value: " << NYT::NodeToYsonString(dataNode));
- value = Base64Decode(dataNode.AsList().front().AsString());
+ YQL_ENSURE(dataNode.IsList() && dataNode.AsList().size() == 1 && dataNode.AsList().front().IsString(), "Unexpected atom value: " << NYT::NodeToYsonString(dataNode));
+ value = Base64Decode(dataNode.AsList().front().AsString());
}
- return ctx.NewAtom(node->Pos(), value);
+ return ctx.NewAtom(node->Pos(), value);
}
TScopedAlloc alloc;
TTypeEnvironment env(alloc);
TStringStream err;
TProgramBuilder pgmBuilder(env, functionRegistry);
- TType* mkqlType = NCommon::BuildType(*clonedArg->GetTypeAnn(), pgmBuilder, err);
+ TType* mkqlType = NCommon::BuildType(*clonedArg->GetTypeAnn(), pgmBuilder, err);
if (!mkqlType) {
ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to process type: " << err.Str()));
return nullptr;
@@ -1040,7 +1040,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
}
if (isTypePipeline) {
- auto yson = TStringBuf(value->AsStringRef());
+ auto yson = TStringBuf(value->AsStringRef());
auto type = NCommon::ParseTypeFromYson(yson, ctx, ctx.GetPosition(node->Pos()));
if (!type) {
return nullptr;
@@ -1060,8 +1060,8 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
if (input.IsCallable("WorldArg")) {
YQL_ENSURE(input.ChildrenSize() == 1 && input.Child(0)->IsAtom());
auto index = FromString<ui32>(input.Child(0)->Content());
- YQL_ENSURE(index < marked.ExternalWorldsList.size());
- YQL_ENSURE(replaces.emplace(&input, marked.ExternalWorldsList[index]).second);
+ YQL_ENSURE(index < marked.ExternalWorldsList.size());
+ YQL_ENSURE(replaces.emplace(&input, marked.ExternalWorldsList[index]).second);
}
return true;
@@ -1073,7 +1073,7 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
return result;
}
- return NCommon::ValueToExprLiteral(clonedArg->GetTypeAnn(), *value, ctx, node->Pos());
+ return NCommon::ValueToExprLiteral(clonedArg->GetTypeAnn(), *value, ctx, node->Pos());
}, ctx, settings);
if (status.Level == IGraphTransformer::TStatus::Error) {
@@ -1081,11 +1081,11 @@ IGraphTransformer::TStatus EvaluateExpression(const TExprNode::TPtr& input, TExp
}
if (hasPendingEvaluations != IGraphTransformer::TStatus::Ok) {
- YQL_CLOG(DEBUG, CoreEval) << "EvaluateExpression - has pending evaluations";
+ YQL_CLOG(DEBUG, CoreEval) << "EvaluateExpression - has pending evaluations";
return hasPendingEvaluations;
}
- YQL_CLOG(DEBUG, CoreEval) << "EvaluateExpression - finish";
+ YQL_CLOG(DEBUG, CoreEval) << "EvaluateExpression - finish";
// repeat some steps
ctx.Step.Repeat(TExprStep::ValidateProviders);
ctx.Step.Repeat(TExprStep::Configure);
diff --git a/ydb/library/yql/core/services/yql_eval_params.cpp b/ydb/library/yql/core/services/yql_eval_params.cpp
index dce9df4892..a9d8489f55 100644
--- a/ydb/library/yql/core/services/yql_eval_params.cpp
+++ b/ydb/library/yql/core/services/yql_eval_params.cpp
@@ -93,15 +93,15 @@ bool ExtractParameterTypes(const TExprNode::TPtr& input, TTypeAnnotationContext&
}
auto name = param->Child(0)->Content();
- if (!param->GetTypeAnn()) {
+ if (!param->GetTypeAnn()) {
ctx.AddError(TIssue(ctx.GetPosition(param->Pos()), TStringBuilder() << "Failed to type check parameter: " << name));
return nullptr;
}
auto& type = paramTypes[name];
if (!type) {
- type = param->GetTypeAnn();
- } else if (!IsSameAnnotation(*type, *param->GetTypeAnn())) {
+ type = param->GetTypeAnn();
+ } else if (!IsSameAnnotation(*type, *param->GetTypeAnn())) {
ctx.AddError(TIssue(ctx.GetPosition(param->Pos()), TStringBuilder() << "Mismatch of types: " << *type << " != " << *param->GetTypeAnn()
<< " for parameter: " << name));
return nullptr;
diff --git a/ydb/library/yql/core/services/yql_out_transformers.cpp b/ydb/library/yql/core/services/yql_out_transformers.cpp
index d8363095f8..768d806861 100644
--- a/ydb/library/yql/core/services/yql_out_transformers.cpp
+++ b/ydb/library/yql/core/services/yql_out_transformers.cpp
@@ -5,7 +5,7 @@
#include <library/cpp/yson/writer.h>
#include <util/stream/null.h>
-#include <util/stream/str.h>
+#include <util/stream/str.h>
namespace NYql {
@@ -18,7 +18,7 @@ IGraphTransformer::TStatus TExprOutputTransformer::operator()(
return IGraphTransformer::TStatus::Ok;
}
- auto ast = ConvertToAst(*ExprRoot_, ctx, WithTypes_ ? TExprAnnotationFlags::Types : TExprAnnotationFlags::None, true);
+ auto ast = ConvertToAst(*ExprRoot_, ctx, WithTypes_ ? TExprAnnotationFlags::Types : TExprAnnotationFlags::None, true);
ui32 prettyFlags = TAstPrintFlags::ShortQuote;
if (!WithTypes_) {
prettyFlags |= TAstPrintFlags::PerLine;
@@ -47,18 +47,18 @@ IGraphTransformer::TStatus TPlanOutputTransformer::operator()(
return IGraphTransformer::TStatus::Ok;
}
-IGraphTransformer::TStatus TExprLogTransformer::operator()(
- const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx)
-{
- Y_UNUSED(ctx);
- output = input;
- if (YQL_CVLOG_ACTIVE(Level, Component)) {
- TStringStream out;
- auto ast = ConvertToAst(*input, ctx, TExprAnnotationFlags::None, true);
- ast.Root->PrettyPrintTo(out, TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine);
- YQL_CVLOG(Level, Component) << Description << ":\n" << out.Str();
- }
- return IGraphTransformer::TStatus::Ok;
-}
-
-}
+IGraphTransformer::TStatus TExprLogTransformer::operator()(
+ const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx)
+{
+ Y_UNUSED(ctx);
+ output = input;
+ if (YQL_CVLOG_ACTIVE(Level, Component)) {
+ TStringStream out;
+ auto ast = ConvertToAst(*input, ctx, TExprAnnotationFlags::None, true);
+ ast.Root->PrettyPrintTo(out, TAstPrintFlags::ShortQuote | TAstPrintFlags::PerLine);
+ YQL_CVLOG(Level, Component) << Description << ":\n" << out.Str();
+ }
+ return IGraphTransformer::TStatus::Ok;
+}
+
+}
diff --git a/ydb/library/yql/core/services/yql_out_transformers.h b/ydb/library/yql/core/services/yql_out_transformers.h
index d90f88cd99..c88b54c0eb 100644
--- a/ydb/library/yql/core/services/yql_out_transformers.h
+++ b/ydb/library/yql/core/services/yql_out_transformers.h
@@ -39,10 +39,10 @@ private:
class TPlanOutputTransformer {
public:
TPlanOutputTransformer(
- IOutputStream* directOut,
+ IOutputStream* directOut,
IPlanBuilder& builder,
NYson::EYsonFormat outputFormat)
- : DirectOut_(directOut)
+ : DirectOut_(directOut)
, Builder_(builder)
, OutputFormat_(outputFormat)
{
@@ -51,40 +51,40 @@ public:
IGraphTransformer::TStatus operator()(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx);
static TAutoPtr <IGraphTransformer> Sync(
- IOutputStream* directOut,
- IPlanBuilder& builder,
+ IOutputStream* directOut,
+ IPlanBuilder& builder,
NYson::EYsonFormat outputFormat)
{
- return CreateFunctorTransformer(TPlanOutputTransformer(directOut, builder, outputFormat));
+ return CreateFunctorTransformer(TPlanOutputTransformer(directOut, builder, outputFormat));
}
private:
- IOutputStream* DirectOut_;
+ IOutputStream* DirectOut_;
IPlanBuilder& Builder_;
NYson::EYsonFormat OutputFormat_;
};
-class TExprLogTransformer {
-public:
- TExprLogTransformer(const TString& description, NYql::NLog::EComponent component, NYql::NLog::ELevel level)
- : Description(description)
- , Component(component)
- , Level(level) {}
-
- NYql::IGraphTransformer::TStatus operator()(const NYql::TExprNode::TPtr& input, NYql::TExprNode::TPtr& output,
- NYql::TExprContext& ctx);
-
- static TAutoPtr<NYql::IGraphTransformer> Sync(const TString& description,
- NYql::NLog::EComponent component = NYql::NLog::EComponent::Core,
- NYql::NLog::ELevel level = NYql::NLog::ELevel::TRACE)
- {
- return CreateFunctorTransformer(TExprLogTransformer(description, component, level));
- }
-
-private:
- TString Description;
- NYql::NLog::EComponent Component;
- NYql::NLog::ELevel Level;
-};
-
-} // namespace NYql
+class TExprLogTransformer {
+public:
+ TExprLogTransformer(const TString& description, NYql::NLog::EComponent component, NYql::NLog::ELevel level)
+ : Description(description)
+ , Component(component)
+ , Level(level) {}
+
+ NYql::IGraphTransformer::TStatus operator()(const NYql::TExprNode::TPtr& input, NYql::TExprNode::TPtr& output,
+ NYql::TExprContext& ctx);
+
+ static TAutoPtr<NYql::IGraphTransformer> Sync(const TString& description,
+ NYql::NLog::EComponent component = NYql::NLog::EComponent::Core,
+ NYql::NLog::ELevel level = NYql::NLog::ELevel::TRACE)
+ {
+ return CreateFunctorTransformer(TExprLogTransformer(description, component, level));
+ }
+
+private:
+ TString Description;
+ NYql::NLog::EComponent Component;
+ NYql::NLog::ELevel Level;
+};
+
+} // namespace NYql
diff --git a/ydb/library/yql/core/services/yql_plan.cpp b/ydb/library/yql/core/services/yql_plan.cpp
index 4d7cfffaf5..194360b81b 100644
--- a/ydb/library/yql/core/services/yql_plan.cpp
+++ b/ydb/library/yql/core/services/yql_plan.cpp
@@ -111,7 +111,7 @@ struct TProviderInfo {
};
using TProviderInfoMap = TMap<TString, TProviderInfo>;
-
+
void WriteProviders(const TString& tag, const TProviderInfoMap& providers, NYson::TYsonWriter& writer) {
writer.OnKeyedItem(tag);
writer.OnBeginList();
@@ -191,7 +191,7 @@ public:
TNodeMap<TNodeInfo> nodes;
TExprNode::TListType order;
- TProviderInfoMap providers;
+ TProviderInfoMap providers;
writer.OnBeginMap();
writer.OnKeyedItem("Detailed");
@@ -242,8 +242,8 @@ public:
outputs.resize(LIMIT_PINS, TPinInfo(nullptr, nullptr, nullptr, "", true));
}
- WritePins("Inputs", inputs, writer, info.Inputs, providers);
- WritePins("Outputs", outputs, writer, info.Outputs, providers);
+ WritePins("Inputs", inputs, writer, info.Inputs, providers);
+ WritePins("Outputs", outputs, writer, info.Outputs, providers);
info.Provider->GetPlanFormatter().WritePlanDetails(*info.Node, writer);
}
@@ -269,7 +269,7 @@ public:
writer.OnEndList();
writer.OnKeyedItem("OperationRoot");
writer.OnUint64Scalar(lastId);
- WriteProviders("Providers", providers, writer);
+ WriteProviders("Providers", providers, writer);
writer.OnKeyedItem("OperationStats");
writer.OnBeginMap();
for (auto& x : opStats) {
@@ -335,7 +335,7 @@ public:
return;
}
- auto& translatedId = Types_.NodeToOperationId[node->UniqueId()];
+ auto& translatedId = Types_.NodeToOperationId[node->UniqueId()];
if (translatedId == 0) {
translatedId = ++NextId_;
}
@@ -345,21 +345,21 @@ public:
if (node->Content() == CommitName) {
dependencies.push_back(node->Child(0));
auto dataSinkName = node->Child(1)->Child(0)->Content();
- auto datasink = Types_.DataSinkMap.FindPtr(dataSinkName);
+ auto datasink = Types_.DataSinkMap.FindPtr(dataSinkName);
YQL_ENSURE(datasink);
info.Provider = (*datasink).Get();
info.IsVisible = dataSinkName != ResultProviderName;
}
else if (node->ChildrenSize() >= 2 && node->Child(1)->IsCallable("DataSource")) {
auto dataSourceName = node->Child(1)->Child(0)->Content();
- auto datasource = Types_.DataSourceMap.FindPtr(dataSourceName);
+ auto datasource = Types_.DataSourceMap.FindPtr(dataSourceName);
YQL_ENSURE(datasource);
info.Provider = (*datasource).Get();
info.IsVisible = (*datasource)->GetPlanFormatter().GetDependencies(*node, dependencies, true);
}
else if (node->ChildrenSize() >= 2 && node->Child(1)->IsCallable("DataSink")) {
auto dataSinkName = node->Child(1)->Child(0)->Content();
- auto datasink = Types_.DataSinkMap.FindPtr(dataSinkName);
+ auto datasink = Types_.DataSinkMap.FindPtr(dataSinkName);
YQL_ENSURE(datasink);
info.Provider = (*datasink).Get();
info.IsVisible = (*datasink)->GetPlanFormatter().GetDependencies(*node, dependencies, true);
@@ -372,8 +372,8 @@ public:
YQL_ENSURE(provider);
info.Provider = (*provider).Get();
info.IsVisible = (*provider)->GetPlanFormatter().GetDependencies(*node, dependencies, true);
- } else {
- info.IsVisible = false;
+ } else {
+ info.IsVisible = false;
for (auto& child : node->Children()) {
dependencies.push_back(child.Get());
}
@@ -549,21 +549,21 @@ public:
}
}
- void UpdateProviders(TProviderInfoMap& providers, const TExprNode* node, IDataProvider* provider) {
+ void UpdateProviders(TProviderInfoMap& providers, const TExprNode* node, IDataProvider* provider) {
auto path = provider->GetPlanFormatter().GetProviderPath(*node);
- if (providers.FindPtr(path)) {
- return;
- }
-
- ui32 providerId = 0;
- if (auto p = ProviderIds_.FindPtr(path)) {
- providerId = *p;
- } else {
- providerId = static_cast<ui32>(ProviderIds_.size() + 1);
- ProviderIds_.insert({path, providerId});
+ if (providers.FindPtr(path)) {
+ return;
}
- providers.insert(std::make_pair(path, TProviderInfo(providerId, node, provider)));
+ ui32 providerId = 0;
+ if (auto p = ProviderIds_.FindPtr(path)) {
+ providerId = *p;
+ } else {
+ providerId = static_cast<ui32>(ProviderIds_.size() + 1);
+ ProviderIds_.insert({path, providerId});
+ }
+
+ providers.insert(std::make_pair(path, TProviderInfo(providerId, node, provider)));
}
void UpdateProviders(TProviderInfoMap& providers, const TVector<TPinInfo>& pins) {
@@ -602,7 +602,7 @@ public:
YQL_ENSURE(false, "Expected either datasource or sink");
}
- TProviderInfo& FindProvider(TProviderInfoMap& providers, const TPinInfo& pin) const {
+ TProviderInfo& FindProvider(TProviderInfoMap& providers, const TPinInfo& pin) const {
if (pin.DataSource) {
auto providerName = pin.DataSource->Child(0)->Content();
auto providerPtr = Types_.DataSourceMap.FindPtr(providerName);
@@ -627,12 +627,12 @@ public:
void WritePins(const TString& tag, const TVector<TPinInfo>& pins, NYson::TYsonWriter& writer,
TVector<TPinAttrs>& pinAttrs, TProviderInfoMap& providers) {
if (!pins.empty()) {
- UpdateProviders(providers, pins);
+ UpdateProviders(providers, pins);
writer.OnKeyedItem(tag);
writer.OnBeginList();
for (auto pin : pins) {
TPinAttrs attrs(pin);
- auto& info = FindProvider(providers, pin);
+ auto& info = FindProvider(providers, pin);
attrs.ProviderId = info.Id;
writer.OnListItem();
writer.OnBeginList();
@@ -658,7 +658,7 @@ public:
}
void Clear() override {
- Types_.NodeToOperationId.clear();
+ Types_.NodeToOperationId.clear();
PinMap_.clear();
ProviderIds_.clear();
NextId_ = 0;
diff --git a/ydb/library/yql/core/services/yql_transform_pipeline.cpp b/ydb/library/yql/core/services/yql_transform_pipeline.cpp
index ae425f34c8..a312eeda51 100644
--- a/ydb/library/yql/core/services/yql_transform_pipeline.cpp
+++ b/ydb/library/yql/core/services/yql_transform_pipeline.cpp
@@ -31,19 +31,19 @@ TTransformationPipeline& TTransformationPipeline::Add(TAutoPtr<IGraphTransformer
return *this;
}
-TTransformationPipeline& TTransformationPipeline::Add(IGraphTransformer& transformer, const TString& stageName,
- EYqlIssueCode issueCode, const TString& issueMessage)
-{
- Transformers_.push_back(TTransformStage(transformer, stageName, issueCode, issueMessage));
- return *this;
-}
-
-TTransformationPipeline& TTransformationPipeline::AddServiceTransformers(EYqlIssueCode issueCode) {
+TTransformationPipeline& TTransformationPipeline::Add(IGraphTransformer& transformer, const TString& stageName,
+ EYqlIssueCode issueCode, const TString& issueMessage)
+{
+ Transformers_.push_back(TTransformStage(transformer, stageName, issueCode, issueMessage));
+ return *this;
+}
+
+TTransformationPipeline& TTransformationPipeline::AddServiceTransformers(EYqlIssueCode issueCode) {
Transformers_.push_back(TTransformStage(CreateGcNodeTransformer(), "GC", issueCode));
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddParametersEvaluation(const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, EYqlIssueCode issueCode) {
+TTransformationPipeline& TTransformationPipeline::AddParametersEvaluation(const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, EYqlIssueCode issueCode) {
auto& typeCtx = *TypeAnnotationContext_;
Transformers_.push_back(TTransformStage(CreateFunctorTransformer(
[&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
@@ -54,7 +54,7 @@ TTransformationPipeline& TTransformationPipeline::AddParametersEvaluation(const
}
TTransformationPipeline& TTransformationPipeline::AddExpressionEvaluation(const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry,
- IGraphTransformer* calcTransfomer, EYqlIssueCode issueCode) {
+ IGraphTransformer* calcTransfomer, EYqlIssueCode issueCode) {
auto typeCtx = TypeAnnotationContext_;
auto funcReg = &functionRegistry;
Transformers_.push_back(TTransformStage(CreateFunctorTransformer(
@@ -65,7 +65,7 @@ TTransformationPipeline& TTransformationPipeline::AddExpressionEvaluation(const
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddPreTypeAnnotation(EYqlIssueCode issueCode) {
+TTransformationPipeline& TTransformationPipeline::AddPreTypeAnnotation(EYqlIssueCode issueCode) {
auto& typeCtx = *TypeAnnotationContext_;
Transformers_.push_back(TTransformStage(CreateFunctorTransformer(&ExpandApply), "ExpandApply",
issueCode));
@@ -77,24 +77,24 @@ TTransformationPipeline& TTransformationPipeline::AddPreTypeAnnotation(EYqlIssue
Transformers_.push_back(TTransformStage(
CreateConfigureTransformer(*TypeAnnotationContext_), "Configure", issueCode));
- return *this;
-}
-
-TTransformationPipeline& TTransformationPipeline::AddPreIOAnnotation(EYqlIssueCode issueCode) {
+ return *this;
+}
+
+TTransformationPipeline& TTransformationPipeline::AddPreIOAnnotation(EYqlIssueCode issueCode) {
Transformers_.push_back(TTransformStage(
CreateIODiscoveryTransformer(*TypeAnnotationContext_), "IODiscovery", issueCode));
- Transformers_.push_back(TTransformStage(
- CreateEpochsTransformer(*TypeAnnotationContext_), "Epochs", issueCode));
+ Transformers_.push_back(TTransformStage(
+ CreateEpochsTransformer(*TypeAnnotationContext_), "Epochs", issueCode));
AddIntentDeterminationTransformer();
-
- return *this;
-}
-
-TTransformationPipeline& TTransformationPipeline::AddIOAnnotation(EYqlIssueCode issueCode) {
- AddPreIOAnnotation(issueCode);
+
+ return *this;
+}
+
+TTransformationPipeline& TTransformationPipeline::AddIOAnnotation(EYqlIssueCode issueCode) {
+ AddPreIOAnnotation(issueCode);
AddTableMetadataLoaderTransformer();
- auto& typeCtx = *TypeAnnotationContext_;
+ auto& typeCtx = *TypeAnnotationContext_;
Transformers_.push_back(TTransformStage(CreateFunctorTransformer(
[&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
return RewriteIO(input, output, typeCtx, ctx);
@@ -103,7 +103,7 @@ TTransformationPipeline& TTransformationPipeline::AddIOAnnotation(EYqlIssueCode
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddTypeAnnotation(EYqlIssueCode issueCode) {
+TTransformationPipeline& TTransformationPipeline::AddTypeAnnotation(EYqlIssueCode issueCode) {
AddTypeAnnotationTransformer();
Transformers_.push_back(TTransformStage(
CreateFunctorTransformer(&CheckWholeProgramType),
@@ -111,11 +111,11 @@ TTransformationPipeline& TTransformationPipeline::AddTypeAnnotation(EYqlIssueCod
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddPostTypeAnnotation(bool forSubGraph, EYqlIssueCode issueCode) {
- Transformers_.push_back(TTransformStage(
- CreateConstraintTransformer(*TypeAnnotationContext_, false, forSubGraph), "Constraints", issueCode));
- Transformers_.push_back(TTransformStage(
- CreateFunctorTransformer(
+TTransformationPipeline& TTransformationPipeline::AddPostTypeAnnotation(bool forSubGraph, EYqlIssueCode issueCode) {
+ Transformers_.push_back(TTransformStage(
+ CreateConstraintTransformer(*TypeAnnotationContext_, false, forSubGraph), "Constraints", issueCode));
+ Transformers_.push_back(TTransformStage(
+ CreateFunctorTransformer(
[](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
return UpdateCompletness(input, output, ctx);
}
@@ -128,13 +128,13 @@ TTransformationPipeline& TTransformationPipeline::AddPostTypeAnnotation(bool for
return EliminateCommonSubExpressions(input, output, ctx, forSubGraph, *coStore);
}
),
- "CSEE",
- issueCode));
-
- return *this;
-}
-
-TTransformationPipeline& TTransformationPipeline::AddCommonOptimization(EYqlIssueCode issueCode) {
+ "CSEE",
+ issueCode));
+
+ return *this;
+}
+
+TTransformationPipeline& TTransformationPipeline::AddCommonOptimization(EYqlIssueCode issueCode) {
// auto instantCallableTransformer =
// CreateExtCallableTypeAnnotationTransformer(*TypeAnnotationContext_, true);
// TypeAnnotationContext_->CustomInstantTypeTransformer =
@@ -146,7 +146,7 @@ TTransformationPipeline& TTransformationPipeline::AddCommonOptimization(EYqlIssu
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddFinalCommonOptimization(EYqlIssueCode issueCode) {
+TTransformationPipeline& TTransformationPipeline::AddFinalCommonOptimization(EYqlIssueCode issueCode) {
Transformers_.push_back(TTransformStage(
CreateCommonOptFinalTransformer(TypeAnnotationContext_.Get()),
"FinalCommonOptimization",
@@ -154,13 +154,13 @@ TTransformationPipeline& TTransformationPipeline::AddFinalCommonOptimization(EYq
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddOptimization(bool checkWorld, bool withFinalOptimization, EYqlIssueCode issueCode) {
- AddCommonOptimization(issueCode);
+TTransformationPipeline& TTransformationPipeline::AddOptimization(bool checkWorld, bool withFinalOptimization, EYqlIssueCode issueCode) {
+ AddCommonOptimization(issueCode);
Transformers_.push_back(TTransformStage(
CreateRecaptureDataProposalsInspector(*TypeAnnotationContext_, TString{DqProviderName}),
- "RecaptureDataProposals",
- issueCode));
- Transformers_.push_back(TTransformStage(
+ "RecaptureDataProposals",
+ issueCode));
+ Transformers_.push_back(TTransformStage(
CreateLogicalDataProposalsInspector(*TypeAnnotationContext_),
"LogicalDataProposals",
issueCode));
@@ -173,13 +173,13 @@ TTransformationPipeline& TTransformationPipeline::AddOptimization(bool checkWorl
"PhysicalFinalizers",
issueCode));
if (withFinalOptimization) {
- AddFinalCommonOptimization(issueCode);
+ AddFinalCommonOptimization(issueCode);
}
- AddCheckExecution(checkWorld, issueCode);
+ AddCheckExecution(checkWorld, issueCode);
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddCheckExecution(bool checkWorld, EYqlIssueCode issueCode) {
+TTransformationPipeline& TTransformationPipeline::AddCheckExecution(bool checkWorld, EYqlIssueCode issueCode) {
Transformers_.push_back(TTransformStage(
CreateCheckExecutionTransformer(*TypeAnnotationContext_, checkWorld),
"CheckExecution",
@@ -187,15 +187,15 @@ TTransformationPipeline& TTransformationPipeline::AddCheckExecution(bool checkWo
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddRun(TOperationProgressWriter writer, EYqlIssueCode issueCode) {
+TTransformationPipeline& TTransformationPipeline::AddRun(TOperationProgressWriter writer, EYqlIssueCode issueCode) {
Transformers_.push_back(TTransformStage(
- CreateExecutionTransformer(*TypeAnnotationContext_, writer),
+ CreateExecutionTransformer(*TypeAnnotationContext_, writer),
"Execution",
issueCode));
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddIntentDeterminationTransformer(EYqlIssueCode issueCode) {
+TTransformationPipeline& TTransformationPipeline::AddIntentDeterminationTransformer(EYqlIssueCode issueCode) {
Transformers_.push_back(TTransformStage(
CreateIntentDeterminationTransformer(*TypeAnnotationContext_),
"IntentDetermination",
@@ -203,7 +203,7 @@ TTransformationPipeline& TTransformationPipeline::AddIntentDeterminationTransfor
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddTableMetadataLoaderTransformer(EYqlIssueCode issueCode) {
+TTransformationPipeline& TTransformationPipeline::AddTableMetadataLoaderTransformer(EYqlIssueCode issueCode) {
Transformers_.push_back(TTransformStage(
CreateTableMetadataLoader(*TypeAnnotationContext_),
"TableMetadataLoader",
@@ -212,7 +212,7 @@ TTransformationPipeline& TTransformationPipeline::AddTableMetadataLoaderTransfor
}
TTransformationPipeline& TTransformationPipeline::AddTypeAnnotationTransformer(
- TAutoPtr<IGraphTransformer> callableTransformer, EYqlIssueCode issueCode)
+ TAutoPtr<IGraphTransformer> callableTransformer, EYqlIssueCode issueCode)
{
Transformers_.push_back(TTransformStage(
CreateTypeAnnotationTransformer(callableTransformer, *TypeAnnotationContext_),
@@ -221,20 +221,20 @@ TTransformationPipeline& TTransformationPipeline::AddTypeAnnotationTransformer(
return *this;
}
-TTransformationPipeline& TTransformationPipeline::AddTypeAnnotationTransformer(EYqlIssueCode issueCode)
+TTransformationPipeline& TTransformationPipeline::AddTypeAnnotationTransformer(EYqlIssueCode issueCode)
{
auto callableTransformer = CreateExtCallableTypeAnnotationTransformer(*TypeAnnotationContext_);
- return AddTypeAnnotationTransformer(callableTransformer, issueCode);
+ return AddTypeAnnotationTransformer(callableTransformer, issueCode);
}
TAutoPtr<IGraphTransformer> TTransformationPipeline::Build(bool useIssueScopes) {
return CreateCompositeGraphTransformer(Transformers_, useIssueScopes);
}
-TAutoPtr<IGraphTransformer> TTransformationPipeline::BuildWithNoArgChecks(bool useIssueScopes) {
- return CreateCompositeGraphTransformerWithNoArgChecks(Transformers_, useIssueScopes);
-}
-
+TAutoPtr<IGraphTransformer> TTransformationPipeline::BuildWithNoArgChecks(bool useIssueScopes) {
+ return CreateCompositeGraphTransformerWithNoArgChecks(Transformers_, useIssueScopes);
+}
+
TIntrusivePtr<TTypeAnnotationContext> TTransformationPipeline::GetTypeAnnotationContext() const {
return TypeAnnotationContext_;
}
diff --git a/ydb/library/yql/core/services/yql_transform_pipeline.h b/ydb/library/yql/core/services/yql_transform_pipeline.h
index 7e2f5c911e..4bbf229d7b 100644
--- a/ydb/library/yql/core/services/yql_transform_pipeline.h
+++ b/ydb/library/yql/core/services/yql_transform_pipeline.h
@@ -22,33 +22,33 @@ class TTransformationPipeline
public:
TTransformationPipeline(TIntrusivePtr<TTypeAnnotationContext> ctx);
- TTransformationPipeline& AddServiceTransformers(EYqlIssueCode issueCode = TIssuesIds::CORE_GC);
- TTransformationPipeline& AddParametersEvaluation(const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, EYqlIssueCode issueCode = TIssuesIds::CORE_PARAM_EVALUATION);
- TTransformationPipeline& AddPreTypeAnnotation(EYqlIssueCode issueCode = TIssuesIds::CORE_PRE_TYPE_ANN);
+ TTransformationPipeline& AddServiceTransformers(EYqlIssueCode issueCode = TIssuesIds::CORE_GC);
+ TTransformationPipeline& AddParametersEvaluation(const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry, EYqlIssueCode issueCode = TIssuesIds::CORE_PARAM_EVALUATION);
+ TTransformationPipeline& AddPreTypeAnnotation(EYqlIssueCode issueCode = TIssuesIds::CORE_PRE_TYPE_ANN);
TTransformationPipeline& AddExpressionEvaluation(const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry,
- IGraphTransformer* calcTransfomer = nullptr, EYqlIssueCode issueCode = TIssuesIds::CORE_EXPR_EVALUATION);
- TTransformationPipeline& AddPreIOAnnotation(EYqlIssueCode issueCode = TIssuesIds::CORE_PRE_TYPE_ANN);
- TTransformationPipeline& AddIOAnnotation(EYqlIssueCode issueCode = TIssuesIds::CORE_PRE_TYPE_ANN);
- TTransformationPipeline& AddTypeAnnotation(EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN);
- TTransformationPipeline& AddPostTypeAnnotation(bool forSubGraph = false, EYqlIssueCode issueCode = TIssuesIds::CORE_POST_TYPE_ANN);
- TTransformationPipeline& AddCommonOptimization(EYqlIssueCode issueCode = TIssuesIds::CORE_OPTIMIZATION);
- TTransformationPipeline& AddFinalCommonOptimization(EYqlIssueCode issueCode = TIssuesIds::CORE_OPTIMIZATION);
- TTransformationPipeline& AddOptimization(bool checkWorld = true, bool withFinalOptimization = true, EYqlIssueCode issueCode = TIssuesIds::CORE_OPTIMIZATION);
- TTransformationPipeline& AddCheckExecution(bool checkWorld = true, EYqlIssueCode issueCode = TIssuesIds::CORE_OPTIMIZATION);
- TTransformationPipeline& AddRun(TOperationProgressWriter writer, EYqlIssueCode issueCode = TIssuesIds::CORE_EXEC);
-
- TTransformationPipeline& AddIntentDeterminationTransformer(EYqlIssueCode issueCode = TIssuesIds::CORE_INTENT);
- TTransformationPipeline& AddTableMetadataLoaderTransformer(EYqlIssueCode issueCode = TIssuesIds::CORE_TABLE_METADATA_LOADER);
- TTransformationPipeline& AddTypeAnnotationTransformer(TAutoPtr<IGraphTransformer> callableTransformer, EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN);
- TTransformationPipeline& AddTypeAnnotationTransformer(EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN);
+ IGraphTransformer* calcTransfomer = nullptr, EYqlIssueCode issueCode = TIssuesIds::CORE_EXPR_EVALUATION);
+ TTransformationPipeline& AddPreIOAnnotation(EYqlIssueCode issueCode = TIssuesIds::CORE_PRE_TYPE_ANN);
+ TTransformationPipeline& AddIOAnnotation(EYqlIssueCode issueCode = TIssuesIds::CORE_PRE_TYPE_ANN);
+ TTransformationPipeline& AddTypeAnnotation(EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN);
+ TTransformationPipeline& AddPostTypeAnnotation(bool forSubGraph = false, EYqlIssueCode issueCode = TIssuesIds::CORE_POST_TYPE_ANN);
+ TTransformationPipeline& AddCommonOptimization(EYqlIssueCode issueCode = TIssuesIds::CORE_OPTIMIZATION);
+ TTransformationPipeline& AddFinalCommonOptimization(EYqlIssueCode issueCode = TIssuesIds::CORE_OPTIMIZATION);
+ TTransformationPipeline& AddOptimization(bool checkWorld = true, bool withFinalOptimization = true, EYqlIssueCode issueCode = TIssuesIds::CORE_OPTIMIZATION);
+ TTransformationPipeline& AddCheckExecution(bool checkWorld = true, EYqlIssueCode issueCode = TIssuesIds::CORE_OPTIMIZATION);
+ TTransformationPipeline& AddRun(TOperationProgressWriter writer, EYqlIssueCode issueCode = TIssuesIds::CORE_EXEC);
+
+ TTransformationPipeline& AddIntentDeterminationTransformer(EYqlIssueCode issueCode = TIssuesIds::CORE_INTENT);
+ TTransformationPipeline& AddTableMetadataLoaderTransformer(EYqlIssueCode issueCode = TIssuesIds::CORE_TABLE_METADATA_LOADER);
+ TTransformationPipeline& AddTypeAnnotationTransformer(TAutoPtr<IGraphTransformer> callableTransformer, EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN);
+ TTransformationPipeline& AddTypeAnnotationTransformer(EYqlIssueCode issueCode = TIssuesIds::CORE_TYPE_ANN);
TTransformationPipeline& Add(TAutoPtr<IGraphTransformer> transformer, const TString& stageName,
EYqlIssueCode issueCode = TIssuesIds::DEFAULT_ERROR, const TString& issueMessage = {});
- TTransformationPipeline& Add(IGraphTransformer& transformer, const TString& stageName,
- EYqlIssueCode issueCode = TIssuesIds::DEFAULT_ERROR, const TString& issueMessage = {});
+ TTransformationPipeline& Add(IGraphTransformer& transformer, const TString& stageName,
+ EYqlIssueCode issueCode = TIssuesIds::DEFAULT_ERROR, const TString& issueMessage = {});
TAutoPtr<IGraphTransformer> Build(bool useIssueScopes = true);
- TAutoPtr<IGraphTransformer> BuildWithNoArgChecks(bool useIssueScopes = true);
+ TAutoPtr<IGraphTransformer> BuildWithNoArgChecks(bool useIssueScopes = true);
TIntrusivePtr<TTypeAnnotationContext> GetTypeAnnotationContext() const;
diff --git a/ydb/library/yql/core/type_ann/type_ann_core.cpp b/ydb/library/yql/core/type_ann/type_ann_core.cpp
index 5846e6cb10..867ebb8b31 100644
--- a/ydb/library/yql/core/type_ann/type_ann_core.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_core.cpp
@@ -27,17 +27,17 @@
#include <util/generic/serialized_enum.h>
#include <util/generic/singleton.h>
-#include <util/generic/strbuf.h>
-#include <util/generic/maybe.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/maybe.h>
#include <util/stream/null.h>
#include <util/string/builder.h>
#include <util/string/cast.h>
#include <util/string/join.h>
#include <util/string/split.h>
-#include <algorithm>
-#include <functional>
-
+#include <algorithm>
+#include <functional>
+
namespace NYql {
TString DotJoin(const TStringBuf& sourceName, const TStringBuf& columnName) {
TStringBuilder sb;
@@ -92,15 +92,15 @@ namespace NTypeAnnImpl {
return isValid;
}
- bool EnsureNotInDiscoveryMode(const TExprNode& input, TExtContext& ctx) {
- if (ctx.Types.DiscoveryMode && ctx.Types.EvaluationInProgress) {
- ctx.Expr.AddError(YqlIssue(ctx.Expr.GetPosition(input.Pos()), TIssuesIds::YQL_NOT_ALLOWED_IN_DISCOVERY,
- TStringBuilder() << input.Content() << " is not allowed in Discovery mode"));
- return false;
- }
- return true;
- }
-
+ bool EnsureNotInDiscoveryMode(const TExprNode& input, TExtContext& ctx) {
+ if (ctx.Types.DiscoveryMode && ctx.Types.EvaluationInProgress) {
+ ctx.Expr.AddError(YqlIssue(ctx.Expr.GetPosition(input.Pos()), TIssuesIds::YQL_NOT_ALLOWED_IN_DISCOVERY,
+ TStringBuilder() << input.Content() << " is not allowed in Discovery mode"));
+ return false;
+ }
+ return true;
+ }
+
TMaybe<TString> SerializeTzComponents(bool isValid, ui64 value, ui16 tzId) {
if (isValid) {
return ToString(value) + "," + ToString(*NKikimr::NMiniKQL::FindTimezoneIANAName(tzId));
@@ -533,7 +533,7 @@ namespace NTypeAnnImpl {
IGraphTransformer::TStatus DataSourceWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
- input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
+ input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
return IGraphTransformer::TStatus::Ok;
}
@@ -581,11 +581,11 @@ namespace NTypeAnnImpl {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Bad atom format for type: "
<< input->Content() << ", value: " << TString{input->Head().Content()}.Quote()));
- return IGraphTransformer::TStatus::Error;
- }
-
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprParamsType>(EDataSlot::Decimal, input->Child(1)->Content(), input->Child(2)->Content()));
- if (!input->GetTypeAnn()->Cast<TDataExprParamsType>()->Validate(input->Pos(), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprParamsType>(EDataSlot::Decimal, input->Child(1)->Content(), input->Child(2)->Content()));
+ if (!input->GetTypeAnn()->Cast<TDataExprParamsType>()->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
return IGraphTransformer::TStatus::Ok;
@@ -728,7 +728,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(NKikimr::NUdf::GetDataSlot(input->Content())));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(NKikimr::NUdf::GetDataSlot(input->Content())));
return IGraphTransformer::TStatus::Ok;
}
}
@@ -752,18 +752,18 @@ namespace NTypeAnnImpl {
if (child->ChildrenSize() == 2) {
if (child->Head().Content() == TStringBuf("epoch") || child->Head().Content() == TStringBuf("commitEpoch")) {
- if (!EnsureAtom(*child->Child(1), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- } else {
+ if (!EnsureAtom(*child->Child(1), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ } else {
if (!EnsureComposable(*child->Child(1), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ return IGraphTransformer::TStatus::Error;
+ }
}
}
}
- input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
+ input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
return IGraphTransformer::TStatus::Ok;
}
@@ -784,7 +784,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(worldType);
+ input->SetTypeAnn(worldType);
return IGraphTransformer::TStatus::Ok;
}
@@ -809,26 +809,26 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus ConsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- Y_UNUSED(output);
- if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ IGraphTransformer::TStatus ConsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ Y_UNUSED(output);
+ if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!EnsureWorldType(input->Head(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- input->SetTypeAnn(ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ input->SetTypeAnn(ctx.Expr.MakeType<TTupleExprType>(TTypeAnnotationNode::TListType{
input->Head().GetTypeAnn(),
input->Tail().GetTypeAnn()
- }));
- return IGraphTransformer::TStatus::Ok;
- }
-
+ }));
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus DataSinkWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
- input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
+ input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
return IGraphTransformer::TStatus::Ok;
}
@@ -905,9 +905,9 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(structType->GetItems()[*pos]->GetItemType());
+ input->SetTypeAnn(structType->GetItems()[*pos]->GetItemType());
if (isOptional && input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional && input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Null) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -1131,7 +1131,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- auto otherType = input->Child(2)->GetTypeAnn();
+ auto otherType = input->Child(2)->GetTypeAnn();
const bool isOptional = otherType->GetKind() == ETypeAnnotationKind::Optional;
auto memberName = input->Child(1)->Content();
const TTypeAnnotationNode* foundFieldType = otherType;
@@ -1164,7 +1164,7 @@ namespace NTypeAnnImpl {
foundFieldType = ctx.Expr.MakeType<TOptionalExprType>(foundFieldType);
}
- input->SetTypeAnn(foundFieldType);
+ input->SetTypeAnn(foundFieldType);
return IGraphTransformer::TStatus::Ok;
}
@@ -1186,7 +1186,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- auto type = structObj->GetTypeAnn();
+ auto type = structObj->GetTypeAnn();
const bool optional = type->GetKind() == ETypeAnnotationKind::Optional;
if (optional) {
type = type->Cast<TOptionalExprType>()->GetItemType();
@@ -1204,8 +1204,8 @@ namespace NTypeAnnImpl {
}
}
- input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(allItems));
- if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
+ input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(allItems));
+ if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -1248,8 +1248,8 @@ namespace NTypeAnnImpl {
}
}
- input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(allItems));
- if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
+ input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(allItems));
+ if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -1311,7 +1311,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = structObj->GetTypeAnn();
+ auto type = structObj->GetTypeAnn();
const bool optional = type->GetKind() == ETypeAnnotationKind::Optional;
if (optional) {
type = type->Cast<TOptionalExprType>()->GetItemType();
@@ -1331,11 +1331,11 @@ namespace NTypeAnnImpl {
}
if (allItems.size() < structExprType->GetSize()) {
- input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(allItems));
+ input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(allItems));
} else {
- input->SetTypeAnn(structExprType);
+ input->SetTypeAnn(structExprType);
}
- if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
+ if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -1357,7 +1357,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = structObj->GetTypeAnn();
+ auto type = structObj->GetTypeAnn();
const bool optional = type->GetKind() == ETypeAnnotationKind::Optional;
if (optional) {
type = type->Cast<TOptionalExprType>()->GetItemType();
@@ -1381,8 +1381,8 @@ namespace NTypeAnnImpl {
}
}
- input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(allItems));
- if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
+ input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(allItems));
+ if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -1462,7 +1462,7 @@ namespace NTypeAnnImpl {
}
}
- auto type = structObj->GetTypeAnn();
+ auto type = structObj->GetTypeAnn();
if (type->GetKind() == ETypeAnnotationKind::Optional) {
type = type->Cast<TOptionalExprType>()->GetItemType();
}
@@ -1567,7 +1567,7 @@ namespace NTypeAnnImpl {
}
if (!flattenByColumns.empty()) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(structObj->Pos()), TStringBuilder() <<
- "Column for flatten \"" << flattenByColumns.begin()->first << "\" does not exist"));
+ "Column for flatten \"" << flattenByColumns.begin()->first << "\" does not exist"));
return IGraphTransformer::TStatus::Error;
}
@@ -1576,9 +1576,9 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
if (allFieldOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(resultStruct));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(resultStruct));
} else {
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultStruct));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultStruct));
}
return IGraphTransformer::TStatus::Ok;
}
@@ -1633,9 +1633,9 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(tupleType->GetItems()[index]);
+ input->SetTypeAnn(tupleType->GetItems()[index]);
if (isOptional && input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional && input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Null) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -1660,11 +1660,11 @@ namespace NTypeAnnImpl {
}
auto memberName = input->Child(1)->Content();
- auto newField = ctx.Expr.MakeType<TItemExprType>(memberName, input->Child(2)->GetTypeAnn());
+ auto newField = ctx.Expr.MakeType<TItemExprType>(memberName, input->Child(2)->GetTypeAnn());
auto newItems = input->Head().GetTypeAnn()->Cast<TStructExprType>()->GetItems();
newItems.push_back(newField);
- input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(newItems));
- if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
+ input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(newItems));
+ if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -1695,16 +1695,16 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(newItems));
- if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
+ input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(newItems));
+ if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus RemovePrefixMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
+ IGraphTransformer::TStatus RemovePrefixMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
auto& firstChild = input->Head();
@@ -1714,145 +1714,145 @@ namespace NTypeAnnImpl {
YQL_ENSURE(firstChild.Type() == TExprNode::Lambda);
ctx.Expr.AddError(TIssue(
ctx.Expr.GetPosition(firstChild.Pos()),
- TStringBuilder() << "Expected struct, variant, or sequence type, but got lambda"
+ TStringBuilder() << "Expected struct, variant, or sequence type, but got lambda"
));
return IGraphTransformer::TStatus::Error;
}
- auto status = NormalizeAtomListForDiveOrSelect(input, output, ctx);
- if (status != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
- const TTypeAnnotationNode* resultType = nullptr;
- bool isSequence = true;
- const TTypeAnnotationNode* itemType = GetItemType(*firstChildType);
- if (!itemType) {
- itemType = firstChildType;
- isSequence = false;
- }
-
- auto prefixes = input->Child(1);
- auto rebuildStructType = [&ctx, prefixes](const TTypeAnnotationNode* structType) {
+ auto status = NormalizeAtomListForDiveOrSelect(input, output, ctx);
+ if (status != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
+ const TTypeAnnotationNode* resultType = nullptr;
+ bool isSequence = true;
+ const TTypeAnnotationNode* itemType = GetItemType(*firstChildType);
+ if (!itemType) {
+ itemType = firstChildType;
+ isSequence = false;
+ }
+
+ auto prefixes = input->Child(1);
+ auto rebuildStructType = [&ctx, prefixes](const TTypeAnnotationNode* structType) {
TVector<const TItemExprType*> newItems;
- for (auto& field : structType->Cast<TStructExprType>()->GetItems()) {
- if (!AnyOf(prefixes->Children(), [field](const auto& prefixNode) { return field->GetName().StartsWith(prefixNode->Content()); })) {
+ for (auto& field : structType->Cast<TStructExprType>()->GetItems()) {
+ if (!AnyOf(prefixes->Children(), [field](const auto& prefixNode) { return field->GetName().StartsWith(prefixNode->Content()); })) {
newItems.push_back(field);
}
}
- return ctx.Expr.MakeType<TStructExprType>(newItems);
- };
-
- if (itemType->GetKind() == ETypeAnnotationKind::Struct) {
- resultType = rebuildStructType(itemType);
- if (resultType == itemType) {
- output = input->HeadPtr();
- return IGraphTransformer::TStatus::Repeat;
- }
- if (!resultType->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- } else if (itemType->GetKind() == ETypeAnnotationKind::Variant) {
- auto varType = itemType->Cast<TVariantExprType>();
- if (varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
- auto tupleType = varType->GetUnderlyingType()->Cast<TTupleExprType>();
- TTypeAnnotationNode::TListType newTupleItems;
- for (size_t i = 0; i < tupleType->GetSize(); ++i) {
- auto tupleItemType = tupleType->GetItems()[i];
- if (tupleItemType->GetKind() != ETypeAnnotationKind::Struct) {
- output = input->HeadPtr();
- return IGraphTransformer::TStatus::Repeat;
- }
- newTupleItems.push_back(rebuildStructType(tupleItemType));
- }
- resultType = ctx.Expr.MakeType<TVariantExprType>(ctx.Expr.MakeType<TTupleExprType>(newTupleItems));
- } else {
- YQL_ENSURE(varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Struct);
- auto structType = varType->GetUnderlyingType()->Cast<TStructExprType>();
- TVector<const TItemExprType*> newStructItems;
- for (size_t i = 0; i < structType->GetSize(); ++i) {
- auto structItemType = structType->GetItems()[i];
- if (structItemType->GetItemType()->GetKind() != ETypeAnnotationKind::Struct) {
- output = input->HeadPtr();
- return IGraphTransformer::TStatus::Repeat;
- }
- newStructItems.push_back(ctx.Expr.MakeType<TItemExprType>(structItemType->GetName(), rebuildStructType(structItemType->GetItemType())));
- }
- resultType = ctx.Expr.MakeType<TVariantExprType>(ctx.Expr.MakeType<TStructExprType>(newStructItems));
- }
- if (resultType == itemType) {
- output = input->HeadPtr();
- return IGraphTransformer::TStatus::Repeat;
- }
+ return ctx.Expr.MakeType<TStructExprType>(newItems);
+ };
+
+ if (itemType->GetKind() == ETypeAnnotationKind::Struct) {
+ resultType = rebuildStructType(itemType);
+ if (resultType == itemType) {
+ output = input->HeadPtr();
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ if (!resultType->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ } else if (itemType->GetKind() == ETypeAnnotationKind::Variant) {
+ auto varType = itemType->Cast<TVariantExprType>();
+ if (varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
+ auto tupleType = varType->GetUnderlyingType()->Cast<TTupleExprType>();
+ TTypeAnnotationNode::TListType newTupleItems;
+ for (size_t i = 0; i < tupleType->GetSize(); ++i) {
+ auto tupleItemType = tupleType->GetItems()[i];
+ if (tupleItemType->GetKind() != ETypeAnnotationKind::Struct) {
+ output = input->HeadPtr();
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ newTupleItems.push_back(rebuildStructType(tupleItemType));
+ }
+ resultType = ctx.Expr.MakeType<TVariantExprType>(ctx.Expr.MakeType<TTupleExprType>(newTupleItems));
+ } else {
+ YQL_ENSURE(varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Struct);
+ auto structType = varType->GetUnderlyingType()->Cast<TStructExprType>();
+ TVector<const TItemExprType*> newStructItems;
+ for (size_t i = 0; i < structType->GetSize(); ++i) {
+ auto structItemType = structType->GetItems()[i];
+ if (structItemType->GetItemType()->GetKind() != ETypeAnnotationKind::Struct) {
+ output = input->HeadPtr();
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ newStructItems.push_back(ctx.Expr.MakeType<TItemExprType>(structItemType->GetName(), rebuildStructType(structItemType->GetItemType())));
+ }
+ resultType = ctx.Expr.MakeType<TVariantExprType>(ctx.Expr.MakeType<TStructExprType>(newStructItems));
+ }
+ if (resultType == itemType) {
+ output = input->HeadPtr();
+ return IGraphTransformer::TStatus::Repeat;
+ }
} else {
- output = input->HeadPtr();
- return IGraphTransformer::TStatus::Repeat;
- }
-
- if (isSequence) {
- resultType = MakeSequenceType(firstChildType->GetKind(), *resultType, ctx.Expr);
- }
- input->SetTypeAnn(resultType);
-
- return IGraphTransformer::TStatus::Ok;
- }
-
- IGraphTransformer::TStatus RemoveSystemMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- auto& firstChild = input->Head();
- auto firstChildType = firstChild.GetTypeAnn();
-
- if (HasError(firstChildType, ctx.Expr) || !firstChildType) {
- YQL_ENSURE(firstChild.Type() == TExprNode::Lambda);
- ctx.Expr.AddError(TIssue(
- ctx.Expr.GetPosition(firstChild.Pos()),
- TStringBuilder() << "Expected struct, variant, or sequence type, but got lambda"
- ));
- return IGraphTransformer::TStatus::Error;
- }
-
- const TTypeAnnotationNode* itemType = GetItemType(*firstChildType);
- if (!itemType) {
- itemType = firstChildType;
- }
- switch (itemType->GetKind()) {
- case ETypeAnnotationKind::Variant: {
- auto varType = itemType->Cast<TVariantExprType>();
- if (varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
- auto tupleType = varType->GetUnderlyingType()->Cast<TTupleExprType>();
- if (AnyOf(tupleType->GetItems(), [](auto tupleItemType) { return tupleItemType->GetKind() != ETypeAnnotationKind::Struct; })) {
- output = input->HeadPtr();
- break;
- }
- } else {
- auto structType = varType->GetUnderlyingType()->Cast<TStructExprType>();
- if (AnyOf(structType->GetItems(), [](auto structItemType) { return structItemType->GetItemType()->GetKind() != ETypeAnnotationKind::Struct; })) {
- output = input->HeadPtr();
- break;
- }
- }
+ output = input->HeadPtr();
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ if (isSequence) {
+ resultType = MakeSequenceType(firstChildType->GetKind(), *resultType, ctx.Expr);
+ }
+ input->SetTypeAnn(resultType);
+
+ return IGraphTransformer::TStatus::Ok;
+ }
+
+ IGraphTransformer::TStatus RemoveSystemMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ auto& firstChild = input->Head();
+ auto firstChildType = firstChild.GetTypeAnn();
+
+ if (HasError(firstChildType, ctx.Expr) || !firstChildType) {
+ YQL_ENSURE(firstChild.Type() == TExprNode::Lambda);
+ ctx.Expr.AddError(TIssue(
+ ctx.Expr.GetPosition(firstChild.Pos()),
+ TStringBuilder() << "Expected struct, variant, or sequence type, but got lambda"
+ ));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ const TTypeAnnotationNode* itemType = GetItemType(*firstChildType);
+ if (!itemType) {
+ itemType = firstChildType;
+ }
+ switch (itemType->GetKind()) {
+ case ETypeAnnotationKind::Variant: {
+ auto varType = itemType->Cast<TVariantExprType>();
+ if (varType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
+ auto tupleType = varType->GetUnderlyingType()->Cast<TTupleExprType>();
+ if (AnyOf(tupleType->GetItems(), [](auto tupleItemType) { return tupleItemType->GetKind() != ETypeAnnotationKind::Struct; })) {
+ output = input->HeadPtr();
+ break;
+ }
+ } else {
+ auto structType = varType->GetUnderlyingType()->Cast<TStructExprType>();
+ if (AnyOf(structType->GetItems(), [](auto structItemType) { return structItemType->GetItemType()->GetKind() != ETypeAnnotationKind::Struct; })) {
+ output = input->HeadPtr();
+ break;
+ }
+ }
[[fallthrough]];
- }
- // passthrough with Struct
- case ETypeAnnotationKind::Struct:
- output = ctx.Expr.Builder(input->Pos())
- .Callable("RemovePrefixMembers")
- .Add(0, input->HeadPtr())
- .List(1)
- .Atom(0, "_yql_", TNodeFlags::Default)
- .Seal()
- .Seal()
- .Build();
- break;
- default:
- output = input->HeadPtr();
- }
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ }
+ // passthrough with Struct
+ case ETypeAnnotationKind::Struct:
+ output = ctx.Expr.Builder(input->Pos())
+ .Callable("RemovePrefixMembers")
+ .Add(0, input->HeadPtr())
+ .List(1)
+ .Atom(0, "_yql_", TNodeFlags::Default)
+ .Seal()
+ .Seal()
+ .Build();
+ break;
+ default:
+ output = input->HeadPtr();
+ }
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
IGraphTransformer::TStatus ReplaceMemberWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
if (!EnsureArgsCount(*input, 3, ctx.Expr)) {
@@ -1879,9 +1879,9 @@ namespace NTypeAnnImpl {
}
auto newItems = input->Head().GetTypeAnn()->Cast<TStructExprType>()->GetItems();
- newItems[*pos] = ctx.Expr.MakeType<TItemExprType>(memberName, input->Child(2)->GetTypeAnn());
- input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(newItems));
- if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
+ newItems[*pos] = ctx.Expr.MakeType<TItemExprType>(memberName, input->Child(2)->GetTypeAnn());
+ input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(newItems));
+ if (!input->GetTypeAnn()->Cast<TStructExprType>()->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -2034,9 +2034,9 @@ namespace NTypeAnnImpl {
}
}
- input->SetTypeAnn(dataType1);
+ input->SetTypeAnn(dataType1);
if (isSomeOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
input->SetUnorderedChildren();
return IGraphTransformer::TStatus::Ok;
@@ -2063,7 +2063,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool));
if (!Order) {
input->SetUnorderedChildren();
}
@@ -2234,7 +2234,7 @@ namespace NTypeAnnImpl {
resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType);
}
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return IGraphTransformer::TStatus::Ok;
}
@@ -2303,7 +2303,7 @@ namespace NTypeAnnImpl {
resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType);
}
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return IGraphTransformer::TStatus::Ok;
}
@@ -2359,7 +2359,7 @@ namespace NTypeAnnImpl {
resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType);
}
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return IGraphTransformer::TStatus::Ok;
}
@@ -2412,7 +2412,7 @@ namespace NTypeAnnImpl {
resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType);
}
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return IGraphTransformer::TStatus::Ok;
}
@@ -2462,7 +2462,7 @@ namespace NTypeAnnImpl {
resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType);
}
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return IGraphTransformer::TStatus::Ok;
}
@@ -2525,7 +2525,7 @@ namespace NTypeAnnImpl {
resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType);
}
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return IGraphTransformer::TStatus::Ok;
}
@@ -2557,7 +2557,7 @@ namespace NTypeAnnImpl {
resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType);
}
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return IGraphTransformer::TStatus::Ok;
}
@@ -2606,7 +2606,7 @@ namespace NTypeAnnImpl {
resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType);
}
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return IGraphTransformer::TStatus::Ok;
}
@@ -2661,7 +2661,7 @@ namespace NTypeAnnImpl {
auto dataSlot = dataType[i]->GetSlot();
if (!IsDataTypeUnsigned(dataSlot)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected unsigned data type, but got: "
- << *input->Child(i)->GetTypeAnn()));
+ << *input->Child(i)->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -2683,7 +2683,7 @@ namespace NTypeAnnImpl {
resultType = ctx.Expr.MakeType<TOptionalExprType>(resultType);
}
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return IGraphTransformer::TStatus::Ok;
}
@@ -2791,13 +2791,13 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(data1);
+ input->SetTypeAnn(data1);
if (dataType1 == EDataSlot::Utf8 && dataType2 == EDataSlot::String) {
- input->SetTypeAnn(data2);
+ input->SetTypeAnn(data2);
}
if (isOptional1 || isOptional2) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -2886,7 +2886,7 @@ namespace NTypeAnnImpl {
return combinedStatus;
}
- input->SetTypeAnn(originalType);
+ input->SetTypeAnn(originalType);
return IGraphTransformer::TStatus::Ok;
}
@@ -3002,7 +3002,7 @@ namespace NTypeAnnImpl {
return convertStatus;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint8)));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint8)));
return IGraphTransformer::TStatus::Ok;
}
@@ -3109,7 +3109,7 @@ namespace NTypeAnnImpl {
}
auto itemType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(itemType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(itemType));
return IGraphTransformer::TStatus::Ok;
}
@@ -3193,9 +3193,9 @@ namespace NTypeAnnImpl {
}
input->SetTypeAnn(ctx.Expr.MakeType<TTaggedExprType>(input->Head().GetTypeAnn(), input->Tail().Content()));
- if (!input->GetTypeAnn()->Cast<TTaggedExprType>()->Validate(input->Pos(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
+ if (!input->GetTypeAnn()->Cast<TTaggedExprType>()->Validate(input->Pos(), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
return IGraphTransformer::TStatus::Ok;
}
@@ -3220,7 +3220,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(tagType->GetBaseType());
+ input->SetTypeAnn(tagType->GetBaseType());
return IGraphTransformer::TStatus::Ok;
}
@@ -3350,9 +3350,9 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(structType);
+ input->SetTypeAnn(structType);
} else {
- input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(TVector<const TItemExprType*>()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TStructExprType>(TVector<const TItemExprType*>()));
}
return IGraphTransformer::TStatus::Ok;
@@ -3410,9 +3410,9 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint32));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint32));
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -3462,7 +3462,7 @@ namespace NTypeAnnImpl {
dataTypeAnn= ctx.Expr.MakeType<TDataExprType>(*slot);
}
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(dataTypeAnn));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(dataTypeAnn));
return IGraphTransformer::TStatus::Ok;
}
@@ -3506,12 +3506,12 @@ namespace NTypeAnnImpl {
dataTypeAnn= ctx.Expr.MakeType<TDataExprType>(*slot);
}
- input->SetTypeAnn(dataTypeAnn);
- if (isDecimal && !input->GetTypeAnn()->Cast<TDataExprParamsType>()->Validate(input->Pos(), ctx.Expr)) {
+ input->SetTypeAnn(dataTypeAnn);
+ if (isDecimal && !input->GetTypeAnn()->Cast<TDataExprParamsType>()->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -3552,8 +3552,8 @@ namespace NTypeAnnImpl {
auto dataTypeAnn = isDecimal ?
ctx.Expr.MakeType<TDataExprParamsType>(*slot, input->Child(2)->Content(), input->Child(3)->Content()):
ctx.Expr.MakeType<TDataExprType>(*slot);
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(dataTypeAnn));
- if (isDecimal && !input->GetTypeAnn()->Cast<TDataExprParamsType>()->Validate(input->Pos(), ctx.Expr)) {
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(dataTypeAnn));
+ if (isDecimal && !input->GetTypeAnn()->Cast<TDataExprParamsType>()->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
return IGraphTransformer::TStatus::Ok;
@@ -3633,9 +3633,9 @@ namespace NTypeAnnImpl {
}
const auto dataTypeAnn = ctx.Expr.MakeType<TDataExprType>(targetSlot);
- input->SetTypeAnn(dataTypeAnn);
+ input->SetTypeAnn(dataTypeAnn);
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -3809,7 +3809,7 @@ namespace NTypeAnnImpl {
}
const auto dataTypeAnn = ctx.Expr.MakeType<TDataExprType>(targetSlot);
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(dataTypeAnn));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(dataTypeAnn));
return IGraphTransformer::TStatus::Ok;
}
@@ -3901,25 +3901,25 @@ namespace NTypeAnnImpl {
targetType = input->Tail().GetTypeAnn()->Cast<TTypeExprType>()->GetType();
- if (!EnsureComputableType(input->Tail().Pos(), *targetType, ctx.Expr)) {
+ if (!EnsureComputableType(input->Tail().Pos(), *targetType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
}
- if (!EnsureComputable(input->Head(), ctx.Expr)) {
+ if (!EnsureComputable(input->Head(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
const auto sourceType = input->Head().GetTypeAnn();
const auto options = CastResult<Strong>(sourceType, targetType);
if (!(options & NKikimr::NUdf::ECastOptions::Impossible)) {
- auto type = targetType;
- if (targetType->GetKind() != ETypeAnnotationKind::Optional && (options & NKikimr::NUdf::ECastOptions::MayFail)) {
- if (!EnsurePersistableType(input->Tail().Pos(), *targetType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- type = ctx.Expr.MakeType<TOptionalExprType>(targetType);
- }
+ auto type = targetType;
+ if (targetType->GetKind() != ETypeAnnotationKind::Optional && (options & NKikimr::NUdf::ECastOptions::MayFail)) {
+ if (!EnsurePersistableType(input->Tail().Pos(), *targetType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ type = ctx.Expr.MakeType<TOptionalExprType>(targetType);
+ }
if (ETypeAnnotationKind::Null == type->GetKind()) {
output = ctx.Expr.NewCallable(input->Tail().Pos(), "Null", {});
@@ -4061,9 +4061,9 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp));
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -4092,7 +4092,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(type);
+ input->SetTypeAnn(type);
return IGraphTransformer::TStatus::Ok;
}
@@ -4106,7 +4106,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
@@ -4129,7 +4129,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(type);
+ input->SetTypeAnn(type);
return IGraphTransformer::TStatus::Ok;
}
@@ -4147,7 +4147,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool));
return IGraphTransformer::TStatus::Ok;
}
@@ -4246,7 +4246,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(retType);
+ input->SetTypeAnn(retType);
return IGraphTransformer::TStatus::Ok;
}
@@ -4393,9 +4393,9 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn((isOptional1 || isOptional2) ?
+ input->SetTypeAnn((isOptional1 || isOptional2) ?
(const TTypeAnnotationNode*)ctx.Expr.MakeType<TOptionalExprType>(retDataType) :
- (const TTypeAnnotationNode*)retDataType);
+ (const TTypeAnnotationNode*)retDataType);
} else {
if (!IsDataTypeFloat(dataType2->GetSlot())) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Tail().Pos()), TStringBuilder()
@@ -4406,9 +4406,9 @@ namespace NTypeAnnImpl {
const TDataExprType* retDataType = ctx.Expr.MakeType<TDataExprType>(
(dataType1->GetSlot() == EDataSlot::Double || dataType2->GetSlot() == EDataSlot::Double) ? EDataSlot::Double : EDataSlot::Float);
- input->SetTypeAnn((isOptional1 || isOptional2) ?
+ input->SetTypeAnn((isOptional1 || isOptional2) ?
(const TTypeAnnotationNode*)ctx.Expr.MakeType<TOptionalExprType>(retDataType) :
- (const TTypeAnnotationNode*)retDataType);
+ (const TTypeAnnotationNode*)retDataType);
}
return IGraphTransformer::TStatus::Ok;
@@ -4465,82 +4465,82 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
template <NKikimr::NUdf::EDataSlot DataSlot>
- IGraphTransformer::TStatus DataGeneratorWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) {
+ IGraphTransformer::TStatus DataGeneratorWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) {
Y_UNUSED(output);
- if (!EnsureNotInDiscoveryMode(*input, ctx)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ if (!EnsureNotInDiscoveryMode(*input, ctx)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!EnsureDependsOnTail(*input, ctx.Expr, 0)) {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(DataSlot));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(DataSlot));
input->SetUnorderedChildren();
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus TablePathWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) {
+ IGraphTransformer::TStatus TablePathWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) {
Y_UNUSED(output);
- if (ctx.Types.StrictTableProps) {
- if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- } else {
- if (!EnsureMaxArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- }
- if (input->ChildrenSize() > 0) {
+ if (ctx.Types.StrictTableProps) {
+ if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ } else {
+ if (!EnsureMaxArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+ if (input->ChildrenSize() > 0) {
if (!EnsureDependsOn(input->Head(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- auto depOn = input->Head().HeadPtr();
- if (ctx.Types.StrictTableProps && !EnsureStructType(*depOn, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (depOn->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct) {
- auto structType = depOn->GetTypeAnn()->Cast<TStructExprType>();
- if (auto pos = structType->FindItem("_yql_sys_tablepath")) {
- bool isOptional = false;
- const TDataExprType* dataType = nullptr;
- // Optional type may appear after UnionAll/UnionMerge with empty list
- if (!EnsureDataOrOptionalOfData(depOn->Pos(), structType->GetItems()[*pos]->GetItemType(), isOptional, dataType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!EnsureSpecificDataType(depOn->Pos(), *dataType, NUdf::EDataSlot::String, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- output = ctx.Expr.Builder(input->Pos())
- .Callable("Member")
- .Add(0, depOn)
- .Atom(1, "_yql_sys_tablepath", TNodeFlags::Default)
- .Seal()
- .Build();
- if (isOptional) {
- output = ctx.Expr.Builder(input->Pos())
- .Callable("Coalesce")
- .Add(0, output)
- .Callable(1, "String")
- .Atom(0, "", TNodeFlags::Default)
- .Seal()
- .Seal()
- .Build();
- }
- return IGraphTransformer::TStatus::Repeat;
- }
- }
- }
-
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(NUdf::EDataSlot::String));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ auto depOn = input->Head().HeadPtr();
+ if (ctx.Types.StrictTableProps && !EnsureStructType(*depOn, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (depOn->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct) {
+ auto structType = depOn->GetTypeAnn()->Cast<TStructExprType>();
+ if (auto pos = structType->FindItem("_yql_sys_tablepath")) {
+ bool isOptional = false;
+ const TDataExprType* dataType = nullptr;
+ // Optional type may appear after UnionAll/UnionMerge with empty list
+ if (!EnsureDataOrOptionalOfData(depOn->Pos(), structType->GetItems()[*pos]->GetItemType(), isOptional, dataType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!EnsureSpecificDataType(depOn->Pos(), *dataType, NUdf::EDataSlot::String, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ output = ctx.Expr.Builder(input->Pos())
+ .Callable("Member")
+ .Add(0, depOn)
+ .Atom(1, "_yql_sys_tablepath", TNodeFlags::Default)
+ .Seal()
+ .Build();
+ if (isOptional) {
+ output = ctx.Expr.Builder(input->Pos())
+ .Callable("Coalesce")
+ .Add(0, output)
+ .Callable(1, "String")
+ .Atom(0, "", TNodeFlags::Default)
+ .Seal()
+ .Seal()
+ .Build();
+ }
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ }
+ }
+
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(NUdf::EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
@@ -4578,74 +4578,74 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Repeat;
}
- IGraphTransformer::TStatus TableRecordWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) {
+ IGraphTransformer::TStatus TableRecordWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) {
Y_UNUSED(output);
- if (ctx.Types.StrictTableProps) {
- if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- } else {
- if (!EnsureMaxArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- }
- if (input->ChildrenSize() > 0) {
+ if (ctx.Types.StrictTableProps) {
+ if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ } else {
+ if (!EnsureMaxArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+ if (input->ChildrenSize() > 0) {
if (!EnsureDependsOn(input->Head(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- auto depOn = input->Head().HeadPtr();
- if (ctx.Types.StrictTableProps && !EnsureStructType(*depOn, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (depOn->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct) {
- auto structType = depOn->GetTypeAnn()->Cast<TStructExprType>();
- if (auto pos = structType->FindItem("_yql_sys_tablerecord")) {
- bool isOptional = false;
- const TDataExprType* dataType = nullptr;
- // Optional type may appear after UnionAll/UnionMerge with empty list
- if (!EnsureDataOrOptionalOfData(depOn->Pos(), structType->GetItems()[*pos]->GetItemType(), isOptional, dataType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!EnsureSpecificDataType(depOn->Pos(), *dataType, NUdf::EDataSlot::Uint64, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- output = ctx.Expr.Builder(input->Pos())
- .Callable("Member")
- .Add(0, depOn)
- .Atom(1, "_yql_sys_tablerecord", TNodeFlags::Default)
- .Seal()
- .Build();
- if (isOptional) {
- output = ctx.Expr.Builder(input->Pos())
- .Callable("Coalesce")
- .Add(0, output)
- .Callable(1, "String")
- .Atom(0, "0", TNodeFlags::Default)
- .Seal()
- .Seal()
- .Build();
- }
- return IGraphTransformer::TStatus::Repeat;
- }
- }
- }
-
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(NUdf::EDataSlot::Uint64));
- return IGraphTransformer::TStatus::Ok;
- }
-
- IGraphTransformer::TStatus IsKeySwitchWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- Y_UNUSED(output);
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ auto depOn = input->Head().HeadPtr();
+ if (ctx.Types.StrictTableProps && !EnsureStructType(*depOn, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (depOn->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct) {
+ auto structType = depOn->GetTypeAnn()->Cast<TStructExprType>();
+ if (auto pos = structType->FindItem("_yql_sys_tablerecord")) {
+ bool isOptional = false;
+ const TDataExprType* dataType = nullptr;
+ // Optional type may appear after UnionAll/UnionMerge with empty list
+ if (!EnsureDataOrOptionalOfData(depOn->Pos(), structType->GetItems()[*pos]->GetItemType(), isOptional, dataType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!EnsureSpecificDataType(depOn->Pos(), *dataType, NUdf::EDataSlot::Uint64, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ output = ctx.Expr.Builder(input->Pos())
+ .Callable("Member")
+ .Add(0, depOn)
+ .Atom(1, "_yql_sys_tablerecord", TNodeFlags::Default)
+ .Seal()
+ .Build();
+ if (isOptional) {
+ output = ctx.Expr.Builder(input->Pos())
+ .Callable("Coalesce")
+ .Add(0, output)
+ .Callable(1, "String")
+ .Atom(0, "0", TNodeFlags::Default)
+ .Seal()
+ .Seal()
+ .Build();
+ }
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ }
+ }
+
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(NUdf::EDataSlot::Uint64));
+ return IGraphTransformer::TStatus::Ok;
+ }
+
+ IGraphTransformer::TStatus IsKeySwitchWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ Y_UNUSED(output);
if (!EnsureArgsCount(*input, 4, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!EnsureComputable(input->Head(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!EnsureComputable(*input->ChildPtr(1), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -4677,10 +4677,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool));
- return IGraphTransformer::TStatus::Ok;
- }
-
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool));
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus TableNameWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
@@ -4756,7 +4756,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(type);
+ input->SetTypeAnn(type);
return IGraphTransformer::TStatus::Ok;
}
@@ -4784,7 +4784,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(type);
+ input->SetTypeAnn(type);
return IGraphTransformer::TStatus::Ok;
}
@@ -4800,9 +4800,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -4822,7 +4822,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return status;
}
- auto type = input->Child(2)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(2)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() != ETypeAnnotationKind::Variant) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() << "Expected variant type, but got: "
<< *type));
@@ -4863,7 +4863,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return convertStatus;
}
- input->SetTypeAnn(type);
+ input->SetTypeAnn(type);
return IGraphTransformer::TStatus::Ok;
}
@@ -4946,9 +4946,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -4985,14 +4985,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- if (!IsSameAnnotation(*input->Child(i)->Child(1)->GetTypeAnn(), *payloadType)) {
+ if (!IsSameAnnotation(*input->Child(i)->Child(1)->GetTypeAnn(), *payloadType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Child(1)->Pos()), TStringBuilder()
- << "Mismatch type of payload, expected: " << *payloadType << ", but got: " << *input->Child(i)->Child(1)->GetTypeAnn()));
+ << "Mismatch type of payload, expected: " << *payloadType << ", but got: " << *input->Child(i)->Child(1)->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
}
- input->SetTypeAnn(dictType);
+ input->SetTypeAnn(dictType);
return IGraphTransformer::TStatus::Ok;
}
@@ -5002,12 +5002,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- if constexpr (ContainsOrLookup) {
- if (IsDataOrOptionalOfData(input->Head().GetTypeAnn())) {
- return WithWrapper(input, output, ctx);
- }
- }
-
+ if constexpr (ContainsOrLookup) {
+ if (IsDataOrOptionalOfData(input->Head().GetTypeAnn())) {
+ return WithWrapper(input, output, ctx);
+ }
+ }
+
if (IsNull(input->Head())) {
output = ContainsOrLookup ? MakeBool(input->Pos(), false, ctx.Expr) : input->HeadPtr();
return IGraphTransformer::TStatus::Repeat;
@@ -5058,10 +5058,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
const auto payloadType = InList ? keyType : dictType->Cast<TDictExprType>()->GetPayloadType();
const auto lookupType = input->Tail().GetTypeAnn();
- if (!EnsureEquatableType(input->Head().Pos(), *keyType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ if (!EnsureEquatableType(input->Head().Pos(), *keyType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (NKikimr::NUdf::ECastOptions::Impossible & CastResult<true>(lookupType, keyType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()),
TStringBuilder() << "Can't lookup " << *lookupType << " in set of " << *keyType));
@@ -5266,14 +5266,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
items.push_back(dictType->GetKeyType());
items.push_back(dictType->GetPayloadType());
auto tupleType = ctx.Expr.MakeType<TTupleExprType>(items);
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(tupleType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(tupleType));
} else {
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(Mode == EDictItems::Keys ?
- dictType->GetKeyType() : dictType->GetPayloadType()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(Mode == EDictItems::Keys ?
+ dictType->GetKeyType() : dictType->GetPayloadType()));
}
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -5302,7 +5302,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- auto memberType = valueNode->GetTypeAnn();
+ auto memberType = valueNode->GetTypeAnn();
if (!EnsureComputableType(valueNode->Pos(), *memberType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -5326,7 +5326,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(structType);
+ input->SetTypeAnn(structType);
return IGraphTransformer::TStatus::Ok;
}
@@ -5357,7 +5357,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- if (IsSameAnnotation(*commonType, *child->GetTypeAnn())) {
+ if (IsSameAnnotation(*commonType, *child->GetTypeAnn())) {
continue;
}
@@ -5366,7 +5366,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
auto& arg2 = input->ChildRef(i);
auto item1 = arg1;
auto item2 = arg2;
- if (SilentInferCommonType(item1, *commonType, item2, *arg2->GetTypeAnn(), ctx.Expr, commonType)
+ if (SilentInferCommonType(item1, *commonType, item2, *arg2->GetTypeAnn(), ctx.Expr, commonType)
!= IGraphTransformer::TStatus::Error) {
needRetype = needRetype || (item2 != arg2);
arg2 = item2;
@@ -5384,7 +5384,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
}
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), TStringBuilder() <<
- "Cannot infer common type for : " << *commonType << " and " << *child->GetTypeAnn()));
+ "Cannot infer common type for : " << *commonType << " and " << *child->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -5409,7 +5409,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(dictType);
+ input->SetTypeAnn(dictType);
return IGraphTransformer::TStatus::Ok;
}
@@ -5424,7 +5424,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) {
return status;
}
- const auto keyType = typeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ const auto keyType = typeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureComputableType(typeNode->Pos(), *keyType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -5455,7 +5455,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(dictType);
+ input->SetTypeAnn(dictType);
return IGraphTransformer::TStatus::Ok;
}
@@ -5668,7 +5668,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -5680,7 +5680,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(thenType);
+ input->SetTypeAnn(thenType);
return IGraphTransformer::TStatus::Ok;
}
@@ -6060,8 +6060,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
}
auto param = input->HeadPtr();
- const auto type = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
- const ui32 idx = IsSameAnnotation(*param->GetTypeAnn(), *type) ? 2 : 3;
+ const auto type = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ const ui32 idx = IsSameAnnotation(*param->GetTypeAnn(), *type) ? 2 : 3;
const auto selected = input->Child(idx);
if (!EnsureLambda(*selected, ctx.Expr)) {
@@ -6123,7 +6123,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- auto targetType = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto targetType = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (Strict) {
if (!IsSameAnnotation(*valueType, *targetType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Mismatch types: "
@@ -6132,7 +6132,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
}
} else {
auto arg = ctx.Expr.NewArgument(input->Pos(), "arg");
- auto status = TrySilentConvertTo(arg, *valueType, *targetType, ctx.Expr, TConvertFlags().Set(NConvertFlags::AllowUnsafeConvert));
+ auto status = TrySilentConvertTo(arg, *valueType, *targetType, ctx.Expr, TConvertFlags().Set(NConvertFlags::AllowUnsafeConvert));
if (status.Level == IGraphTransformer::TStatus::Error) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Cannot convert type "
<< *valueType << " into " << *targetType << message));
@@ -6197,28 +6197,28 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Repeat;
}
- IGraphTransformer::TStatus TupleSizeAssertWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!EnsureAtom(*input->Child(1), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- ui32 tupleSize = 0;
- if (!TryFromString(input->Child(1)->Content(), tupleSize)) {
+ IGraphTransformer::TStatus TupleSizeAssertWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!EnsureAtom(*input->Child(1), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ ui32 tupleSize = 0;
+ if (!TryFromString(input->Child(1)->Content(), tupleSize)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Expected number, but got: "
- << input->Child(1)->Content()));
- return IGraphTransformer::TStatus::Error;
- }
+ << input->Child(1)->Content()));
+ return IGraphTransformer::TStatus::Error;
+ }
auto nodePtr = input->HeadPtr();
- if (!EnsureTupleTypeSize(*nodePtr, tupleSize, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- output = nodePtr;
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ if (!EnsureTupleTypeSize(*nodePtr, tupleSize, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ output = nodePtr;
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
IGraphTransformer::TStatus EnsureWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
if (!EnsureMinMaxArgsCount(*input, 2, 3, ctx.Expr)) {
@@ -6280,9 +6280,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
auto ui64Type = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64);
auto dictType = ctx.Expr.MakeType<TDictExprType>(ui64Type, listType->GetItemType());
- input->SetTypeAnn(dictType);
+ input->SetTypeAnn(dictType);
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -6349,13 +6349,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- if (!lambda1->GetTypeAnn() || !lambda2->GetTypeAnn()) {
+ if (!lambda1->GetTypeAnn() || !lambda2->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
TMaybe<bool> isMany;
TMaybe<bool> isHashed;
- TMaybe<ui64> itemsCount;
+ TMaybe<ui64> itemsCount;
bool isCompact;
TMaybe<TIssue> error = ParseToDictSettings(*input, ctx.Expr, isMany, isHashed, itemsCount, isCompact);
if (error) {
@@ -6363,8 +6363,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- auto keyType = lambda1->GetTypeAnn();
- auto payloadType = lambda2->GetTypeAnn();
+ auto keyType = lambda1->GetTypeAnn();
+ auto payloadType = lambda2->GetTypeAnn();
if (*isMany) {
payloadType = ctx.Expr.MakeType<TListExprType>(payloadType);
}
@@ -6374,23 +6374,23 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- if (*isHashed) {
- if (!keyType->IsEquatable() || !keyType->IsHashable()) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
- << "Expected equatable and hashable key type for hashed dict, but got: " << *keyType));
- return IGraphTransformer::TStatus::Error;
- }
- } else {
- if (!keyType->IsComparable()) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
- << "Expected comparable key type for sorted dict, but got: " << *keyType));
- return IGraphTransformer::TStatus::Error;
- }
+ if (*isHashed) {
+ if (!keyType->IsEquatable() || !keyType->IsHashable()) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
+ << "Expected equatable and hashable key type for hashed dict, but got: " << *keyType));
+ return IGraphTransformer::TStatus::Error;
+ }
+ } else {
+ if (!keyType->IsComparable()) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
+ << "Expected comparable key type for sorted dict, but got: " << *keyType));
+ return IGraphTransformer::TStatus::Error;
+ }
}
- input->SetTypeAnn(dictType);
+ input->SetTypeAnn(dictType);
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -6476,7 +6476,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TVoidExprType>());
+ input->SetTypeAnn(ctx.Expr.MakeType<TVoidExprType>());
return IGraphTransformer::TStatus::Ok;
}
@@ -6486,7 +6486,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TNullExprType>());
+ input->SetTypeAnn(ctx.Expr.MakeType<TNullExprType>());
return IGraphTransformer::TStatus::Ok;
}
@@ -6526,7 +6526,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(type);
+ input->SetTypeAnn(type);
return IGraphTransformer::TStatus::Ok;
}
@@ -6565,13 +6565,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*lambda->GetTypeAnn(), *innerType)) {
+ if (!IsSameAnnotation(*lambda->GetTypeAnn(), *innerType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Mismatch lambda return type, "
- << *lambda->GetTypeAnn() << " != " << *innerType));
+ << *lambda->GetTypeAnn() << " != " << *innerType));
return IGraphTransformer::TStatus::Error;
}
@@ -6611,11 +6611,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureArgsCount(*input, 1, ctx.Expr) || !ValidateFileAlias(input->Head(), ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!EnsureNotInDiscoveryMode(*input, ctx)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ if (!EnsureNotInDiscoveryMode(*input, ctx)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
@@ -6624,10 +6624,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureArgsCount(*input, 1, ctx.Expr) || !ValidateFolderAlias(input->Head(), ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!EnsureNotInDiscoveryMode(*input, ctx)) {
- return IGraphTransformer::TStatus::Error;
- }
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ if (!EnsureNotInDiscoveryMode(*input, ctx)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
@@ -6636,10 +6636,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureArgsCount(*input, 1, ctx.Expr) || !ValidateFileAlias(input->Head(), ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!EnsureNotInDiscoveryMode(*input, ctx)) {
- return IGraphTransformer::TStatus::Error;
- }
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ if (!EnsureNotInDiscoveryMode(*input, ctx)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
@@ -6653,12 +6653,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TStructExprType>(TVector<const TItemExprType*>{
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TStructExprType>(TVector<const TItemExprType*>{
ctx.Expr.MakeType<TItemExprType>("Name", ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)),
ctx.Expr.MakeType<TItemExprType>("IsFolder", ctx.Expr.MakeType<TDataExprType>(EDataSlot::Bool)),
ctx.Expr.MakeType<TItemExprType>("Url", ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String))),
ctx.Expr.MakeType<TItemExprType>("Path", ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)))
- })));
+ })));
return IGraphTransformer::TStatus::Ok;
}
@@ -6669,11 +6669,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TStructExprType>(TVector<const TItemExprType*>{
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TStructExprType>(TVector<const TItemExprType*>{
ctx.Expr.MakeType<TItemExprType>("Name", ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)),
ctx.Expr.MakeType<TItemExprType>("Category", ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)),
ctx.Expr.MakeType<TItemExprType>("Subcategory", ctx.Expr.MakeType<TDataExprType>(EDataSlot::String))
- })));
+ })));
return IGraphTransformer::TStatus::Ok;
}
@@ -6715,7 +6715,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return status;
}
- userType = input->Child(2)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ userType = input->Child(2)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (userType->GetKind() == ETypeAnnotationKind::Void) {
userType = nullptr;
}
@@ -6739,7 +6739,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return status;
}
- auto type = input->Child(4)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(4)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureCallableType(input->Child(4)->Pos(), *type, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -6754,7 +6754,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return status;
}
- cachedRunConfigType = input->Child(5)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ cachedRunConfigType = input->Child(5)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
}
// (6) file alias
@@ -6808,10 +6808,10 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
TVector<IUdfResolver::TFunction*> functions;
functions.push_back(&description);
- TIssueScopeGuard issueScope(ctx.Expr.IssueManager, [input, &ctx]() -> TIssuePtr {
- return MakeIntrusive<TIssue>(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "At " << input->Head().Content());
- });
-
+ TIssueScopeGuard issueScope(ctx.Expr.IssueManager, [input, &ctx]() -> TIssuePtr {
+ return MakeIntrusive<TIssue>(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "At " << input->Head().Content());
+ });
+
if (!ctx.LoadUdfMetadata(functions)) {
return IGraphTransformer::TStatus::Error;
}
@@ -6873,9 +6873,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
static const std::unordered_map<std::string_view, std::string_view> deprecated = {
{"String.Reverse", "'Unicode::Reverse'"},
- {"String.ToLower", "'String::AsciiToLower' or 'Unicode::ToLower'"},
- {"String.ToUpper", "'String::AsciiToUpper' or 'Unicode::ToUpper'"},
- {"String.ToTitle", "'String::AsciiToTitle' or 'Unicode::ToTitle'"},
+ {"String.ToLower", "'String::AsciiToLower' or 'Unicode::ToLower'"},
+ {"String.ToUpper", "'String::AsciiToUpper' or 'Unicode::ToUpper'"},
+ {"String.ToTitle", "'String::AsciiToTitle' or 'Unicode::ToTitle'"},
{"String.Substring", "'SUBSTRING' builtin function"},
{"String.Find", "'FIND' builtin function"},
{"String.ReverseFind", "'RFIND' builtin function"},
@@ -6889,7 +6889,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
}
}
- input->SetTypeAnn(cachedType);
+ input->SetTypeAnn(cachedType);
return IGraphTransformer::TStatus::Ok;
}
@@ -6918,7 +6918,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (auto status = EnsureTypeRewrite(input->ChildRef(2), ctx.Expr); status != IGraphTransformer::TStatus::Ok) {
return status;
}
- const TTypeAnnotationNode* tn = input->Child(2)->GetTypeAnn();
+ const TTypeAnnotationNode* tn = input->Child(2)->GetTypeAnn();
const TTypeExprType* type = tn->Cast<TTypeExprType>();
if (!EnsureCallableType(input->Child(2)->Pos(), *type->GetType(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
@@ -6945,78 +6945,78 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
scriptType = NKikimr::NMiniKQL::CanonizeScriptType(scriptType);
auto canonizedModuleName = NKikimr::NMiniKQL::ScriptTypeAsStr(scriptType);
bool isCustomPython = NKikimr::NMiniKQL::IsCustomPython(scriptType);
- bool foundModule = false;
-
- // resolve script udf from external resources (files / urls)
- // (main usage of CustomPython)
- {
- TVector<IUdfResolver::TFunction*> functions;
-
- if (!ctx.LoadUdfMetadata(functions)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- foundModule = ctx.Types.UdfModules.find(canonizedModuleName) != ctx.Types.UdfModules.end();
- }
+ bool foundModule = false;
+
+ // resolve script udf from external resources (files / urls)
+ // (main usage of CustomPython)
+ {
+ TVector<IUdfResolver::TFunction*> functions;
+
+ if (!ctx.LoadUdfMetadata(functions)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ foundModule = ctx.Types.UdfModules.find(canonizedModuleName) != ctx.Types.UdfModules.end();
+ }
// fallback for preinstalled CustomPython case
- if (!foundModule) {
- foundModule = static_cast<bool>(ctx.Types.UdfResolver->GetSystemModulePath(canonizedModuleName));
- }
-
- if (!foundModule) {
- if (isCustomPython) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
- << "Module with CustomPython UDF not found\n"
- "Provide it via url / file link or add it to your YQL installation"));
- } else {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
- << "Module not loaded for script type: " << canonizedModuleName));
- }
- return IGraphTransformer::TStatus::Error;
- }
-
- if (input->ChildrenSize() == 5) {
- if (!EnsureTuple(*input->Child(4), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- for (auto setting: input->Child(4)->Children()) {
- if (!EnsureTupleMinSize(*setting, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- auto nameNode = setting->Child(0);
- if (!EnsureAtom(*nameNode, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (nameNode->Content() == "cpu") {
- if (!EnsureTupleSize(*setting, 2, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- double val = 0.;
- if (!TryFromString(setting->Child(1)->Content(), val) || val == 0.) {
+ if (!foundModule) {
+ foundModule = static_cast<bool>(ctx.Types.UdfResolver->GetSystemModulePath(canonizedModuleName));
+ }
+
+ if (!foundModule) {
+ if (isCustomPython) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
+ << "Module with CustomPython UDF not found\n"
+ "Provide it via url / file link or add it to your YQL installation"));
+ } else {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
+ << "Module not loaded for script type: " << canonizedModuleName));
+ }
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (input->ChildrenSize() == 5) {
+ if (!EnsureTuple(*input->Child(4), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ for (auto setting: input->Child(4)->Children()) {
+ if (!EnsureTupleMinSize(*setting, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ auto nameNode = setting->Child(0);
+ if (!EnsureAtom(*nameNode, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (nameNode->Content() == "cpu") {
+ if (!EnsureTupleSize(*setting, 2, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ double val = 0.;
+ if (!TryFromString(setting->Child(1)->Content(), val) || val == 0.) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(nameNode->Pos()), TStringBuilder()
<< "Bad " << TString{nameNode->Content()}.Quote() << " setting value: " << setting->Child(1)->Content()));
- return IGraphTransformer::TStatus::Error;
- }
- }
- else if (nameNode->Content() == "extraMem") {
- ui64 val = 0;
- if (!TryFromString(setting->Child(1)->Content(), val)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+ else if (nameNode->Content() == "extraMem") {
+ ui64 val = 0;
+ if (!TryFromString(setting->Child(1)->Content(), val)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(nameNode->Pos()), TStringBuilder()
<< "Bad " << TString{nameNode->Content()}.Quote() << " setting value: " << setting->Child(1)->Content()));
- return IGraphTransformer::TStatus::Error;
- }
- }
- else {
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+ else {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(nameNode->Pos()), TStringBuilder() << "Unsupported setting: " << nameNode->Content()));
- return IGraphTransformer::TStatus::Error;
- }
- }
- }
-
- input->SetTypeAnn(callableType);
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+ }
+
+ input->SetTypeAnn(callableType);
return IGraphTransformer::TStatus::Ok;
}
@@ -7054,7 +7054,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
for (ui32 i = 1; i < input->ChildrenSize(); ++i) {
const auto& arg = type->GetArguments()[i - 1];
const bool isAutoMap = arg.Flags & NKikimr::NUdf::ICallablePayload::TArgumentFlags::AutoMap;
- const auto srcType = input->Child(i)->GetTypeAnn();
+ const auto srcType = input->Child(i)->GetTypeAnn();
if (HasError(srcType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -7063,8 +7063,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (convertStatus.Level == IGraphTransformer::TStatus::Error) {
if (isAutoMap && srcType && (srcType->GetKind() == ETypeAnnotationKind::Optional || srcType->GetKind() == ETypeAnnotationKind::Null)) {
if (srcType->GetKind() == ETypeAnnotationKind::Null) {
- auto retType = type->GetReturnType();
- input->SetTypeAnn(retType);
+ auto retType = type->GetReturnType();
+ input->SetTypeAnn(retType);
if (retType->GetKind() != ETypeAnnotationKind::Optional) {
retType = ctx.Expr.MakeType<TOptionalExprType>(retType);
}
@@ -7132,7 +7132,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(type->GetReturnType());
+ input->SetTypeAnn(type->GetReturnType());
return IGraphTransformer::TStatus::Ok;
}
@@ -7164,13 +7164,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- auto tupleType = input->Child(1)->GetTypeAnn()->Cast<TTupleExprType>();
+ auto tupleType = input->Child(1)->GetTypeAnn()->Cast<TTupleExprType>();
auto tupleSize = tupleType->GetSize();
if (!EnsureStructType(*input->Child(2), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- auto structType = input->Child(2)->GetTypeAnn()->Cast<TStructExprType>();
+ auto structType = input->Child(2)->GetTypeAnn()->Cast<TStructExprType>();
auto structSize = structType->GetSize();
auto totalArgs = tupleSize + structSize;
if (totalArgs > type->GetArgumentsSize()) {
@@ -7219,7 +7219,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() <<
"Mismatch positional argument types, expected: " <<
*static_cast<const TTypeAnnotationNode*>(expectedTupleType) << ", but got: " <<
- *input->Child(1)->GetTypeAnn()));
+ *input->Child(1)->GetTypeAnn()));
} else {
convertStatus = IGraphTransformer::TStatus::Ok;
}
@@ -7299,7 +7299,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder() <<
"Mismatch named argument types, expected: " <<
*static_cast<const TTypeAnnotationNode*>(expectedStructType) << ", but got: " <<
- *input->Child(2)->GetTypeAnn()));
+ *input->Child(2)->GetTypeAnn()));
} else {
convertStatus = IGraphTransformer::TStatus::Ok;
}
@@ -7396,7 +7396,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(type->GetReturnType());
+ input->SetTypeAnn(type->GetReturnType());
return IGraphTransformer::TStatus::Ok;
}
@@ -7550,17 +7550,17 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
}
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*lambda->GetTypeAnn(), *callableType->GetReturnType())) {
+ if (!IsSameAnnotation(*lambda->GetTypeAnn(), *callableType->GetReturnType())) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of lambda return type: "
- << *lambda->GetTypeAnn() << " != " << *callableType->GetReturnType()));
+ << *lambda->GetTypeAnn() << " != " << *callableType->GetReturnType()));
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(callableType);
+ input->SetTypeAnn(callableType);
return IGraphTransformer::TStatus::Ok;
}
@@ -7581,7 +7581,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return convertStatus;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TResourceExprType>("MTRand"));
+ input->SetTypeAnn(ctx.Expr.MakeType<TResourceExprType>("MTRand"));
return IGraphTransformer::TStatus::Ok;
}
@@ -7598,7 +7598,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
TTypeAnnotationNode::TListType tupleItems(2);
tupleItems[0] = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64);
tupleItems[1] = input->Head().GetTypeAnn();
- input->SetTypeAnn(ctx.Expr.MakeType<TTupleExprType>(tupleItems));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTupleExprType>(tupleItems));
return IGraphTransformer::TStatus::Ok;
}
@@ -7616,7 +7616,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return status;
}
- auto type = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() != ETypeAnnotationKind::Struct) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder() << "Expected struct type, but got: "
<< *type));
@@ -7641,7 +7641,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
}
}
- input->SetTypeAnn(type);
+ input->SetTypeAnn(type);
return IGraphTransformer::TStatus::Ok;
}
@@ -7686,7 +7686,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(tupleType->GetItems()[index]);
+ input->SetTypeAnn(tupleType->GetItems()[index]);
} else {
auto structType = variantType->GetUnderlyingType()->Cast<TStructExprType>();
auto pos = FindOrReportMissingMember(input->Child(1)->Content(), input->Pos(), *structType, ctx);
@@ -7694,70 +7694,70 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(structType->GetItems()[*pos]->GetItemType());
+ input->SetTypeAnn(structType->GetItems()[*pos]->GetItemType());
}
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus VariantItemWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- Y_UNUSED(output);
- if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- bool isOptional = false;
- const TVariantExprType* variantType;
+ IGraphTransformer::TStatus VariantItemWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ Y_UNUSED(output);
+ if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ bool isOptional = false;
+ const TVariantExprType* variantType;
if (input->Head().GetTypeAnn() && input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
- isOptional = true;
+ isOptional = true;
auto itemType = input->Head().GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType();
if (!EnsureVariantType(input->Head().Pos(), *itemType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- variantType = itemType->Cast<TVariantExprType>();
- }
- else {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ variantType = itemType->Cast<TVariantExprType>();
+ }
+ else {
if (!EnsureVariantType(input->Head(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
variantType = input->Head().GetTypeAnn()->Cast<TVariantExprType>();
- }
-
- if (variantType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
- auto tupleType = variantType->GetUnderlyingType()->Cast<TTupleExprType>();
- auto firstType = tupleType->GetItems()[0];
- for (size_t i = 1; i < tupleType->GetSize(); ++i) {
- if (firstType != tupleType->GetItems()[i]) {
+ }
+
+ if (variantType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
+ auto tupleType = variantType->GetUnderlyingType()->Cast<TTupleExprType>();
+ auto firstType = tupleType->GetItems()[0];
+ for (size_t i = 1; i < tupleType->GetSize(); ++i) {
+ if (firstType != tupleType->GetItems()[i]) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
- << "All Variant item types should be equal: " << GetTypeDiff(*firstType, *tupleType->GetItems()[i])));
- return IGraphTransformer::TStatus::Error;
- }
- }
-
- input->SetTypeAnn(firstType);
- } else {
- auto structType = variantType->GetUnderlyingType()->Cast<TStructExprType>();
- auto firstType = structType->GetItems()[0]->GetItemType();
- for (size_t i = 1; i < structType->GetSize(); ++i) {
- if (firstType != structType->GetItems()[i]->GetItemType()) {
+ << "All Variant item types should be equal: " << GetTypeDiff(*firstType, *tupleType->GetItems()[i])));
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
+ input->SetTypeAnn(firstType);
+ } else {
+ auto structType = variantType->GetUnderlyingType()->Cast<TStructExprType>();
+ auto firstType = structType->GetItems()[0]->GetItemType();
+ for (size_t i = 1; i < structType->GetSize(); ++i) {
+ if (firstType != structType->GetItems()[i]->GetItemType()) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder()
- << "All Variant item types should be equal: " << GetTypeDiff(*firstType, *structType->GetItems()[i]->GetItemType())));
- return IGraphTransformer::TStatus::Error;
- }
- }
-
- input->SetTypeAnn(firstType);
- }
-
- if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
- }
- return IGraphTransformer::TStatus::Ok;
- }
-
+ << "All Variant item types should be equal: " << GetTypeDiff(*firstType, *structType->GetItems()[i]->GetItemType())));
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
+ input->SetTypeAnn(firstType);
+ }
+
+ if (isOptional) {
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ }
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus VisitWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
if (!EnsureMinArgsCount(*input, 2, ctx.Expr)) {
@@ -7784,7 +7784,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
const TTypeAnnotationNode* resultType = nullptr;
bool needRepeat = false;
bool hasDefaultValue = false;
- TVector<std::pair<ui32, size_t>> indexOrder;
+ TVector<std::pair<ui32, size_t>> indexOrder;
for (ui32 index = 1; index < input->ChildrenSize(); ++index) {
const TTypeAnnotationNode* currentType = nullptr;
auto child = input->Child(index);
@@ -7828,7 +7828,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- indexOrder.emplace_back(itemIndex, index);
+ indexOrder.emplace_back(itemIndex, index);
usedFields[itemIndex] = true;
++usedCount;
@@ -7849,12 +7849,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
needRepeat = true;
continue;
}
- currentType = lambda->GetTypeAnn();
+ currentType = lambda->GetTypeAnn();
} else {
if (index != input->ChildrenSize() - 1) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Pos()), "Default value should be in the end"));
@@ -7865,7 +7865,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- currentType = child->GetTypeAnn();
+ currentType = child->GetTypeAnn();
hasDefaultValue = true;
}
@@ -7889,27 +7889,27 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- auto less = [](const std::pair<ui32, size_t>& left, const std::pair<ui32, size_t>& right) {
- return left.first < right.first;
- };
-
- if (!IsSorted(indexOrder.begin(), indexOrder.end(), less)) {
- Sort(indexOrder.begin(), indexOrder.end(), less);
- TExprNode::TListType list = input->ChildrenList();
- for (size_t i = 0; i < indexOrder.size(); ++i) {
- list[i * 2 + 1] = input->ChildPtr(indexOrder[i].second);
- list[i * 2 + 2] = input->ChildPtr(indexOrder[i].second + 1);
- }
-
- output = ctx.Expr.ChangeChildren(*input, std::move(list));
- needRepeat = true;
- }
-
+ auto less = [](const std::pair<ui32, size_t>& left, const std::pair<ui32, size_t>& right) {
+ return left.first < right.first;
+ };
+
+ if (!IsSorted(indexOrder.begin(), indexOrder.end(), less)) {
+ Sort(indexOrder.begin(), indexOrder.end(), less);
+ TExprNode::TListType list = input->ChildrenList();
+ for (size_t i = 0; i < indexOrder.size(); ++i) {
+ list[i * 2 + 1] = input->ChildPtr(indexOrder[i].second);
+ list[i * 2 + 2] = input->ChildPtr(indexOrder[i].second + 1);
+ }
+
+ output = ctx.Expr.ChangeChildren(*input, std::move(list));
+ needRepeat = true;
+ }
+
if (needRepeat) {
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return IGraphTransformer::TStatus::Ok;
}
@@ -7939,14 +7939,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
}
if (variantType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint32));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint32));
}
else {
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Utf8));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Utf8));
}
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -7973,7 +7973,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- const TTypeAnnotationNode* unpacked = RemoveOptionalType(input->Child(1)->GetTypeAnn());
+ const TTypeAnnotationNode* unpacked = RemoveOptionalType(input->Child(1)->GetTypeAnn());
bool isYson = false;
bool isYsonNode = false;
bool isJson = false;
@@ -8083,7 +8083,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
output = ctx.Expr.NewCallable(input->Pos(), "Guess", { input->ChildPtr(1), input->ChildPtr(2) });
} else {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), TStringBuilder()
- << "Expected (optional) tuple or variant based on it, but got: " << *input->Child(1)->GetTypeAnn()));
+ << "Expected (optional) tuple or variant based on it, but got: " << *input->Child(1)->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
}
@@ -8113,7 +8113,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
output = ctx.Expr.NewCallable(input->Pos(), "Null", {});
} else {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(1)->Pos()), TStringBuilder()
- << "Expected (optional) list or dict, but got: " << *input->Child(1)->GetTypeAnn()));
+ << "Expected (optional) list or dict, but got: " << *input->Child(1)->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
} else {
@@ -8124,113 +8124,113 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Repeat;
}
- IGraphTransformer::TStatus SqlProcessWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureMinArgsCount(*input, 3, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- const size_t lastPos = input->ChildrenSize() - 1;
- if (!EnsureAtom(*input->Child(lastPos), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- size_t listArg = 0;
- if (!TryFromString<size_t>(input->Child(lastPos)->Content(), listArg) || listArg >= input->ChildrenSize() - 2) {
+ IGraphTransformer::TStatus SqlProcessWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ if (!EnsureMinArgsCount(*input, 3, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ const size_t lastPos = input->ChildrenSize() - 1;
+ if (!EnsureAtom(*input->Child(lastPos), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ size_t listArg = 0;
+ if (!TryFromString<size_t>(input->Child(lastPos)->Content(), listArg) || listArg >= input->ChildrenSize() - 2) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(lastPos)->Pos()), TStringBuilder()
- << "Invalid value of list argument position: " << input->Child(lastPos)->Content()));
- return IGraphTransformer::TStatus::Error;
- }
-
- TExprNode::TListType applyChildren = input->ChildrenList();
- applyChildren.pop_back(); // Remove position of list argument
-
+ << "Invalid value of list argument position: " << input->Child(lastPos)->Content()));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ TExprNode::TListType applyChildren = input->ChildrenList();
+ applyChildren.pop_back(); // Remove position of list argument
+
if (input->Head().Type() != TExprNode::Lambda) {
const TCallableExprType* callableType = input->Head().GetTypeAnn()->Cast<TCallableExprType>();
-
- if (applyChildren.size() < callableType->GetArgumentsSize() + 1 - callableType->GetOptionalArgumentsCount()) {
+
+ if (applyChildren.size() < callableType->GetArgumentsSize() + 1 - callableType->GetOptionalArgumentsCount()) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Invalid number of arguments "
- << (applyChildren.size() - 1) << " to use with callable type " << FormatType(callableType)));
- return IGraphTransformer::TStatus::Error;
- }
-
- const bool expectList = callableType->GetArguments()[listArg].Type->GetKind() == ETypeAnnotationKind::List;
- if (expectList) {
+ << (applyChildren.size() - 1) << " to use with callable type " << FormatType(callableType)));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ const bool expectList = callableType->GetArguments()[listArg].Type->GetKind() == ETypeAnnotationKind::List;
+ if (expectList) {
auto issue = TIssue(ctx.Expr.GetPosition(input->Head().Pos()), "The Udf used in PROCESS accepts List argument type, which prevents some optimizations."
- " Consider to rewrite it using Stream argument type");
- SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_NON_STREAM_BATCH_UDF, issue);
- if (!ctx.Expr.AddWarning(issue)) {
- return IGraphTransformer::TStatus::Error;
- }
- }
-
- output = ctx.Expr.Builder(input->Pos())
- .Callable("OrderedLMap")
- .Add(0, applyChildren[listArg + 1])
- .Lambda(1)
- .Param("stream")
- .Callable("ToSequence")
- .Callable(0, "Apply")
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- for (size_t i = 0; i < applyChildren.size(); ++i) {
- if (i != listArg + 1) {
- parent.Add(i, applyChildren[i]);
- } else if (expectList) {
- parent.Callable(i, "ForwardList")
- .Arg(0, "stream")
- .Seal();
- } else {
- parent.Arg(i, "stream");
- }
- }
- return parent;
- })
- .Seal()
- .Seal()
- .Seal()
- .Seal()
- .Build();
- }
- else {
+ " Consider to rewrite it using Stream argument type");
+ SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_NON_STREAM_BATCH_UDF, issue);
+ if (!ctx.Expr.AddWarning(issue)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
+ output = ctx.Expr.Builder(input->Pos())
+ .Callable("OrderedLMap")
+ .Add(0, applyChildren[listArg + 1])
+ .Lambda(1)
+ .Param("stream")
+ .Callable("ToSequence")
+ .Callable(0, "Apply")
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ for (size_t i = 0; i < applyChildren.size(); ++i) {
+ if (i != listArg + 1) {
+ parent.Add(i, applyChildren[i]);
+ } else if (expectList) {
+ parent.Callable(i, "ForwardList")
+ .Arg(0, "stream")
+ .Seal();
+ } else {
+ parent.Arg(i, "stream");
+ }
+ }
+ return parent;
+ })
+ .Seal()
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+ else {
auto lambda = input->HeadPtr();
- const auto args = lambda->Child(0);
- if (input->ChildrenSize() - 2 != args->ChildrenSize()) {
+ const auto args = lambda->Child(0);
+ if (input->ChildrenSize() - 2 != args->ChildrenSize()) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Different arguments count, lambda has "
- << args->ChildrenSize() << " arguments, but provided " << (input->ChildrenSize() - 2)));
- return IGraphTransformer::TStatus::Error;
- }
-
- output = ctx.Expr.Builder(input->Pos())
- .Callable("OrderedLMap")
- .Add(0, applyChildren[listArg + 1])
- .Lambda(1)
- .Param("stream")
- .Callable("ToSequence")
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- auto replacer = parent.Apply(0, lambda);
- for (size_t i = 1; i < applyChildren.size(); ++i) {
- if (i != listArg + 1) {
- replacer.With(i - 1, applyChildren[i]);
- } else {
- replacer.With(i - 1, "stream");
- }
- }
- replacer.Seal();
- return parent;
- })
- .Seal()
- .Seal()
- .Seal()
- .Build();
- }
-
- return IGraphTransformer::TStatus::Repeat;
- }
-
- IGraphTransformer::TStatus SqlReduceWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ << args->ChildrenSize() << " arguments, but provided " << (input->ChildrenSize() - 2)));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ output = ctx.Expr.Builder(input->Pos())
+ .Callable("OrderedLMap")
+ .Add(0, applyChildren[listArg + 1])
+ .Lambda(1)
+ .Param("stream")
+ .Callable("ToSequence")
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ auto replacer = parent.Apply(0, lambda);
+ for (size_t i = 1; i < applyChildren.size(); ++i) {
+ if (i != listArg + 1) {
+ replacer.With(i - 1, applyChildren[i]);
+ } else {
+ replacer.With(i - 1, "stream");
+ }
+ }
+ replacer.Seal();
+ return parent;
+ })
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ IGraphTransformer::TStatus SqlReduceWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
if (!EnsureArgsCount(*input, 4, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
TPositionHandle pos = input->Pos();
TExprNode::TPtr extractKeyLambda = input->ChildPtr(1);
@@ -8337,27 +8337,27 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureListType(input->Head(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
-
+
TExprNode::TPtr handler;
if (input->Child(2)->Type() != TExprNode::Lambda) {
const auto callableType = input->Child(2)->GetTypeAnn()->Cast<TCallableExprType>();
- if (callableType->GetArgumentsSize() != 2) {
+ if (callableType->GetArgumentsSize() != 2) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "Expected callable with 2 arguments"));
- return IGraphTransformer::TStatus::Error;
- }
-
- const bool expectList = callableType->GetArguments()[1].Type->GetKind() == ETypeAnnotationKind::List;
- if (expectList) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ const bool expectList = callableType->GetArguments()[1].Type->GetKind() == ETypeAnnotationKind::List;
+ if (expectList) {
auto issue = TIssue(ctx.Expr.GetPosition(input->Child(2)->Pos()), "The Udf used in REDUCE accepts List argument type, which prevents some optimizations."
- " Consider to rewrite it using Stream argument type");
- SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_NON_STREAM_BATCH_UDF, issue);
- if (!ctx.Expr.AddWarning(issue)) {
- return IGraphTransformer::TStatus::Error;
- }
- }
-
+ " Consider to rewrite it using Stream argument type");
+ SetIssueCode(EYqlIssueCode::TIssuesIds_EIssueCode_CORE_NON_STREAM_BATCH_UDF, issue);
+ if (!ctx.Expr.AddWarning(issue)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
handler = expectList ?
ctx.Expr.Builder(input->Child(2)->Pos())
.Lambda()
@@ -8374,9 +8374,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
.Add(1, input->TailPtr())
.Seal()
.Seal()
- .Seal()
- .Seal()
- .Seal()
+ .Seal()
+ .Seal()
+ .Seal()
.Seal().Build():
ctx.Expr.Builder(input->Child(2)->Pos())
.Lambda()
@@ -8398,9 +8398,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
} else {
if (const auto& lambda = *input->Child(2); lambda.Head().ChildrenSize() != 2) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda.Pos()), TStringBuilder() << "Expected lambda with 2 arguments"));
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
handler = ctx.Expr.Builder(input->Child(2)->Pos())
.Lambda()
.Param("key")
@@ -8409,18 +8409,18 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
.Callable(0, "ToSequence")
.Apply(0, *input->Child(2))
.With(0, "key")
- .With(1)
- .Callable("Map")
+ .With(1)
+ .Callable("Map")
.Arg(0, "stream")
.Add(1, input->TailPtr())
- .Seal()
- .Done()
- .Seal()
- .Seal()
- .Seal()
+ .Seal()
+ .Done()
+ .Seal()
+ .Seal()
+ .Seal()
.Seal().Build();
- }
-
+ }
+
output = ctx.Expr.Builder(input->Pos())
.Callable("ForwardList")
.Callable(0, "Chopper")
@@ -8445,9 +8445,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
.Seal()
.Seal().Build();
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
// 0 - function kind
// 1 - function name
// 2 - list of pair, settings ("key", value)
@@ -8564,108 +8564,108 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus SqlExtractKeyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- auto status = ConvertToLambda(input->ChildRef(1), ctx.Expr, 1);
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
+ IGraphTransformer::TStatus SqlExtractKeyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ auto status = ConvertToLambda(input->ChildRef(1), ctx.Expr, 1);
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Struct) {
- output = ctx.Expr.Builder(input->Pos())
- .Apply(input->ChildPtr(1))
+ output = ctx.Expr.Builder(input->Pos())
+ .Apply(input->ChildPtr(1))
.With(0, input->HeadPtr())
- .Seal()
- .Build();
- }
+ .Seal()
+ .Build();
+ }
else if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Variant) {
auto underlyingType = input->Head().GetTypeAnn()->Cast<TVariantExprType>()->GetUnderlyingType();
- if (underlyingType->GetKind() == ETypeAnnotationKind::Tuple) {
- auto tupleTypeItems = underlyingType->Cast<TTupleExprType>()->GetItems();
- if (std::adjacent_find(tupleTypeItems.cbegin(), tupleTypeItems.cend(), std::not_equal_to<const TTypeAnnotationNode*>()) == tupleTypeItems.cend()) {
- // All types are the same
- output = ctx.Expr.Builder(input->Pos())
- .Apply(input->ChildPtr(1))
- .With(0)
- .Callable("VariantItem")
+ if (underlyingType->GetKind() == ETypeAnnotationKind::Tuple) {
+ auto tupleTypeItems = underlyingType->Cast<TTupleExprType>()->GetItems();
+ if (std::adjacent_find(tupleTypeItems.cbegin(), tupleTypeItems.cend(), std::not_equal_to<const TTypeAnnotationNode*>()) == tupleTypeItems.cend()) {
+ // All types are the same
+ output = ctx.Expr.Builder(input->Pos())
+ .Apply(input->ChildPtr(1))
+ .With(0)
+ .Callable("VariantItem")
.Add(0, input->HeadPtr())
- .Seal()
- .Done()
- .Seal()
- .Build();
- }
- else { // Non equal types
- output = ctx.Expr.Builder(input->Pos())
- .Callable("Visit")
+ .Seal()
+ .Done()
+ .Seal()
+ .Build();
+ }
+ else { // Non equal types
+ output = ctx.Expr.Builder(input->Pos())
+ .Callable("Visit")
.Add(0, input->HeadPtr())
- .Do([&input, &tupleTypeItems](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- for (size_t i = 0; i < tupleTypeItems.size(); ++i) {
- parent
- .Atom(i * 2 + 1, ToString(i))
- .Lambda(i * 2 + 2)
- .Param("item")
- .Apply(input->ChildPtr(1))
- .With(0, "item")
- .Seal()
- .Seal()
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Build();
- }
- }
- else { // underlyingType->GetKind() == ETypeAnnotationKind::Struct
- auto structTypeItems = underlyingType->Cast<TStructExprType>()->GetItems();
- if (std::adjacent_find(structTypeItems.cbegin(), structTypeItems.cend(),
- [](const TItemExprType* t1, const TItemExprType* t2) { return t1->GetItemType() != t2->GetItemType(); }) == structTypeItems.cend())
- {
- // All types are the same
- output = ctx.Expr.Builder(input->Pos())
- .Apply(input->ChildPtr(1))
- .With(0)
- .Callable("VariantItem")
+ .Do([&input, &tupleTypeItems](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ for (size_t i = 0; i < tupleTypeItems.size(); ++i) {
+ parent
+ .Atom(i * 2 + 1, ToString(i))
+ .Lambda(i * 2 + 2)
+ .Param("item")
+ .Apply(input->ChildPtr(1))
+ .With(0, "item")
+ .Seal()
+ .Seal()
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Build();
+ }
+ }
+ else { // underlyingType->GetKind() == ETypeAnnotationKind::Struct
+ auto structTypeItems = underlyingType->Cast<TStructExprType>()->GetItems();
+ if (std::adjacent_find(structTypeItems.cbegin(), structTypeItems.cend(),
+ [](const TItemExprType* t1, const TItemExprType* t2) { return t1->GetItemType() != t2->GetItemType(); }) == structTypeItems.cend())
+ {
+ // All types are the same
+ output = ctx.Expr.Builder(input->Pos())
+ .Apply(input->ChildPtr(1))
+ .With(0)
+ .Callable("VariantItem")
.Add(0, input->HeadPtr())
- .Seal()
- .Done()
- .Seal()
- .Build();
- }
- else { // Non equal types
- output = ctx.Expr.Builder(input->Pos())
- .Callable("Visit")
+ .Seal()
+ .Done()
+ .Seal()
+ .Build();
+ }
+ else { // Non equal types
+ output = ctx.Expr.Builder(input->Pos())
+ .Callable("Visit")
.Add(0, input->HeadPtr())
- .Do([&input, &structTypeItems](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- for (size_t i = 0; i < structTypeItems.size(); ++i) {
- parent
- .Atom(i * 2 + 1, structTypeItems[i]->GetName())
- .Lambda(i * 2 + 2)
- .Param("item")
- .Apply(input->ChildPtr(1))
- .With(0, "item")
- .Seal()
- .Seal()
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Build();
- }
- }
- }
- else {
+ .Do([&input, &structTypeItems](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ for (size_t i = 0; i < structTypeItems.size(); ++i) {
+ parent
+ .Atom(i * 2 + 1, structTypeItems[i]->GetName())
+ .Lambda(i * 2 + 2)
+ .Param("item")
+ .Apply(input->ChildPtr(1))
+ .With(0, "item")
+ .Seal()
+ .Seal()
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Build();
+ }
+ }
+ }
+ else {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected Struct or Variant type, but got: " << *input->Head().GetTypeAnn()));
- return IGraphTransformer::TStatus::Error;
- }
-
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
IGraphTransformer::TStatus SqlReduceUdfWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
Y_UNUSED(output);
if (!EnsureMinMaxArgsCount(*input, 2, 3, ctx.Expr)) {
@@ -10802,25 +10802,25 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Repeat;
}
- IGraphTransformer::TStatus AutoDemuxListWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ IGraphTransformer::TStatus AutoDemuxListWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!EnsureListType(input->Head(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
auto listType = input->Head().GetTypeAnn()->Cast<TListExprType>();
- if (listType->GetItemType()->GetKind() == ETypeAnnotationKind::Variant) {
- output = ctx.Expr.RenameNode(*input, "Demux");
- } else {
+ if (listType->GetItemType()->GetKind() == ETypeAnnotationKind::Variant) {
+ output = ctx.Expr.RenameNode(*input, "Demux");
+ } else {
output = input->HeadPtr();
- }
-
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ }
+
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
IGraphTransformer::TStatus AggrCountInitWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
@@ -10835,7 +10835,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64));
return IGraphTransformer::TStatus::Ok;
}
@@ -10903,8 +10903,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
}
}
- input->SetTypeAnn(ctx.Expr.MakeType<TResourceExprType>(TStringBuilder() <<
- NKikimr::NMiniKQL::ResourceQueuePrefix << FormatType(queueType)));
+ input->SetTypeAnn(ctx.Expr.MakeType<TResourceExprType>(TStringBuilder() <<
+ NKikimr::NMiniKQL::ResourceQueuePrefix << FormatType(queueType)));
return IGraphTransformer::TStatus::Ok;
}
@@ -10912,7 +10912,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureResourceType(*resourceArg, ctx.Expr)) {
return false;
}
- const auto resourceType = resourceArg->GetTypeAnn()->Cast<TResourceExprType>();
+ const auto resourceType = resourceArg->GetTypeAnn()->Cast<TResourceExprType>();
const auto resourceTag = resourceType->GetTag();
using NKikimr::NMiniKQL::ResourceQueuePrefix;
if (!resourceTag.StartsWith(ResourceQueuePrefix)) {
@@ -10929,7 +10929,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureType(*typeExpr, ctx.Expr)) {
return false;
}
- elementType = typeExpr->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ elementType = typeExpr->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
return true;
}
@@ -10952,7 +10952,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (convertStatus.Level != IGraphTransformer::TStatus::Ok) {
return convertStatus;
}
- input->SetTypeAnn(resourceArg->GetTypeAnn());
+ input->SetTypeAnn(resourceArg->GetTypeAnn());
return IGraphTransformer::TStatus::Ok;
}
@@ -10967,7 +10967,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(resourceArg->GetTypeAnn());
+ input->SetTypeAnn(resourceArg->GetTypeAnn());
return IGraphTransformer::TStatus::Ok;
}
@@ -10989,7 +10989,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return convertStatus;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(expectedValueType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(expectedValueType));
return IGraphTransformer::TStatus::Ok;
}
@@ -11033,7 +11033,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureQueueResource(resourceArg, expectedValueType, ctx)) {
return IGraphTransformer::TStatus::Error;
}
- const TTypeAnnotationNode* streamType = streamArg->GetTypeAnn();
+ const TTypeAnnotationNode* streamType = streamArg->GetTypeAnn();
const TTypeAnnotationNode* itemType = streamType->Cast<TStreamExprType>()->GetItemType();
if (!IsSameAnnotation(*itemType, *expectedValueType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of stream and queue types: "
@@ -11047,7 +11047,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(streamType);
+ input->SetTypeAnn(streamType);
return IGraphTransformer::TStatus::Ok;
}
@@ -11113,7 +11113,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return status;
}
- input->SetTypeAnn(input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType());
+ input->SetTypeAnn(input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType());
return IGraphTransformer::TStatus::Ok;
}
@@ -11128,12 +11128,12 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return slot;
} else {
- if (!node.GetTypeAnn() || node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Type) {
+ if (!node.GetTypeAnn() || node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Type) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either atom or type"));
return {};
}
- auto type = node.GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = node.GetTypeAnn()->Cast<TTypeExprType>()->GetType();
bool isOptional;
const TDataExprType* dataType;
if (!EnsureDataOrOptionalOfData(node.Pos(), type, isOptional, dataType, ctx)) {
@@ -11385,7 +11385,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- if (!NYql::ValidateName(memberNameNode->Pos(), memberNameNode->Content(), "member", ctx.Expr)) {
+ if (!NYql::ValidateName(memberNameNode->Pos(), memberNameNode->Content(), "member", ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -11397,7 +11397,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
}
auto targetTypeExpr = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(*targetSlot));
- input->SetTypeAnn(targetTypeExpr);
+ input->SetTypeAnn(targetTypeExpr);
return IGraphTransformer::TStatus::Ok;
}
@@ -11449,8 +11449,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(*targetSlot));
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(*targetSlot));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
return IGraphTransformer::TStatus::Ok;
}
@@ -11532,8 +11532,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
const auto p1 = tokenName.substr(separator + 1);
auto cred = ctx.Types.FindCredential(p1);
TMaybe<TCredential> clusterCred;
- if (cred == nullptr && p0 == "cluster") {
- if (p1.StartsWith("default_")) {
+ if (cred == nullptr && p0 == "cluster") {
+ if (p1.StartsWith("default_")) {
TStringBuf clusterName = p1;
if (clusterName.SkipPrefix("default_")) {
for (auto& x : ctx.Types.DataSources) {
@@ -11571,169 +11571,169 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus MuxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ IGraphTransformer::TStatus MuxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
const TTypeAnnotationNode* inputType = input->Head().GetTypeAnn();
- const TTypeAnnotationNode* resultType = nullptr;
- if (inputType->GetKind() == ETypeAnnotationKind::Tuple) {
- const TTupleExprType* tupleType = inputType->Cast<TTupleExprType>();
- TTypeAnnotationNode::TListType itemTypes;
- TExprNode::TListType updatedChildren;
- for (size_t i = 0; i < tupleType->GetSize(); ++i) {
+ const TTypeAnnotationNode* resultType = nullptr;
+ if (inputType->GetKind() == ETypeAnnotationKind::Tuple) {
+ const TTupleExprType* tupleType = inputType->Cast<TTupleExprType>();
+ TTypeAnnotationNode::TListType itemTypes;
+ TExprNode::TListType updatedChildren;
+ for (size_t i = 0; i < tupleType->GetSize(); ++i) {
if (!EnsureListType(input->Head().Pos(), *tupleType->GetItems()[i], ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- auto itemType = tupleType->GetItems()[i]->Cast<TListExprType>()->GetItemType();
- if (itemType->GetKind() == ETypeAnnotationKind::Struct
- && AnyOf(itemType->Cast<TStructExprType>()->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) {
-
- if (updatedChildren.empty()) {
- updatedChildren.resize(tupleType->GetSize());
- }
- updatedChildren[i] = ctx.Expr.Builder(input->Head().Pos())
- .Callable("RemovePrefixMembers")
- .Callable(0, "Nth")
- .Add(0, input->HeadPtr())
- .Atom(1, ToString(i), TNodeFlags::Default)
- .Seal()
- .List(1)
- .Atom(0, "_yql_sys_", TNodeFlags::Default)
- .Seal()
- .Seal()
- .Build();
- }
- itemTypes.push_back(itemType);
- }
- if (!updatedChildren.empty()) {
- for (size_t i = 0; i < updatedChildren.size(); ++i) {
- if (!updatedChildren[i]) {
- updatedChildren[i] = ctx.Expr.Builder(input->Head().Pos())
- .Callable("Nth")
- .Add(0, input->HeadPtr())
- .Atom(1, ToString(i), TNodeFlags::Default)
- .Seal()
- .Build();
- }
- }
- output = ctx.Expr.ChangeChild(*input, 0, ctx.Expr.NewList(input->Head().Pos(), std::move(updatedChildren)));
- return IGraphTransformer::TStatus::Repeat;
- }
- resultType = ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TVariantExprType>(ctx.Expr.MakeType<TTupleExprType>(itemTypes)));
- }
- else if (inputType->GetKind() == ETypeAnnotationKind::Struct) {
- const TStructExprType* structType = inputType->Cast<TStructExprType>();
- TVector<const TItemExprType*> itemTypes;
- TExprNode::TListType updatedChildren;
- for (size_t i = 0; i < structType->GetSize(); ++i) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ auto itemType = tupleType->GetItems()[i]->Cast<TListExprType>()->GetItemType();
+ if (itemType->GetKind() == ETypeAnnotationKind::Struct
+ && AnyOf(itemType->Cast<TStructExprType>()->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) {
+
+ if (updatedChildren.empty()) {
+ updatedChildren.resize(tupleType->GetSize());
+ }
+ updatedChildren[i] = ctx.Expr.Builder(input->Head().Pos())
+ .Callable("RemovePrefixMembers")
+ .Callable(0, "Nth")
+ .Add(0, input->HeadPtr())
+ .Atom(1, ToString(i), TNodeFlags::Default)
+ .Seal()
+ .List(1)
+ .Atom(0, "_yql_sys_", TNodeFlags::Default)
+ .Seal()
+ .Seal()
+ .Build();
+ }
+ itemTypes.push_back(itemType);
+ }
+ if (!updatedChildren.empty()) {
+ for (size_t i = 0; i < updatedChildren.size(); ++i) {
+ if (!updatedChildren[i]) {
+ updatedChildren[i] = ctx.Expr.Builder(input->Head().Pos())
+ .Callable("Nth")
+ .Add(0, input->HeadPtr())
+ .Atom(1, ToString(i), TNodeFlags::Default)
+ .Seal()
+ .Build();
+ }
+ }
+ output = ctx.Expr.ChangeChild(*input, 0, ctx.Expr.NewList(input->Head().Pos(), std::move(updatedChildren)));
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ resultType = ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TVariantExprType>(ctx.Expr.MakeType<TTupleExprType>(itemTypes)));
+ }
+ else if (inputType->GetKind() == ETypeAnnotationKind::Struct) {
+ const TStructExprType* structType = inputType->Cast<TStructExprType>();
+ TVector<const TItemExprType*> itemTypes;
+ TExprNode::TListType updatedChildren;
+ for (size_t i = 0; i < structType->GetSize(); ++i) {
if (!EnsureListType(input->Head().Pos(), *structType->GetItems()[i]->GetItemType(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- auto itemType = structType->GetItems()[i]->GetItemType()->Cast<TListExprType>()->GetItemType();
- auto itemName = structType->GetItems()[i]->GetName();
- if (itemType->GetKind() == ETypeAnnotationKind::Struct
- && AnyOf(itemType->Cast<TStructExprType>()->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) {
-
- if (updatedChildren.empty()) {
- updatedChildren.resize(structType->GetSize());
- }
- updatedChildren[i] = ctx.Expr.Builder(input->Head().Pos())
- .List()
- .Atom(0, itemName)
- .Callable(1, "RemovePrefixMembers")
- .Callable(0, "Member")
- .Add(0, input->HeadPtr())
- .Atom(1, itemName)
- .Seal()
- .List(1)
- .Atom(0, "_yql_sys_", TNodeFlags::Default)
- .Seal()
- .Seal()
- .Seal()
- .Build();
- }
-
- itemTypes.push_back(ctx.Expr.MakeType<TItemExprType>(itemName, itemType));
- }
- if (!updatedChildren.empty()) {
- for (size_t i = 0; i < updatedChildren.size(); ++i) {
- if (!updatedChildren[i]) {
- auto itemName = structType->GetItems()[i]->GetName();
- updatedChildren[i] = ctx.Expr.Builder(input->Head().Pos())
- .List()
- .Atom(0, itemName)
- .Callable(1, "Member")
- .Add(0, input->HeadPtr())
- .Atom(1, itemName)
- .Seal()
- .Seal()
- .Build();
- }
- }
- output = ctx.Expr.ChangeChild(*input, 0, ctx.Expr.NewCallable(input->Head().Pos(), "AsStruct", std::move(updatedChildren)));
- return IGraphTransformer::TStatus::Repeat;
- }
- resultType = ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TVariantExprType>(ctx.Expr.MakeType<TStructExprType>(itemTypes)));
- }
- else {
+ return IGraphTransformer::TStatus::Error;
+ }
+ auto itemType = structType->GetItems()[i]->GetItemType()->Cast<TListExprType>()->GetItemType();
+ auto itemName = structType->GetItems()[i]->GetName();
+ if (itemType->GetKind() == ETypeAnnotationKind::Struct
+ && AnyOf(itemType->Cast<TStructExprType>()->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) {
+
+ if (updatedChildren.empty()) {
+ updatedChildren.resize(structType->GetSize());
+ }
+ updatedChildren[i] = ctx.Expr.Builder(input->Head().Pos())
+ .List()
+ .Atom(0, itemName)
+ .Callable(1, "RemovePrefixMembers")
+ .Callable(0, "Member")
+ .Add(0, input->HeadPtr())
+ .Atom(1, itemName)
+ .Seal()
+ .List(1)
+ .Atom(0, "_yql_sys_", TNodeFlags::Default)
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+
+ itemTypes.push_back(ctx.Expr.MakeType<TItemExprType>(itemName, itemType));
+ }
+ if (!updatedChildren.empty()) {
+ for (size_t i = 0; i < updatedChildren.size(); ++i) {
+ if (!updatedChildren[i]) {
+ auto itemName = structType->GetItems()[i]->GetName();
+ updatedChildren[i] = ctx.Expr.Builder(input->Head().Pos())
+ .List()
+ .Atom(0, itemName)
+ .Callable(1, "Member")
+ .Add(0, input->HeadPtr())
+ .Atom(1, itemName)
+ .Seal()
+ .Seal()
+ .Build();
+ }
+ }
+ output = ctx.Expr.ChangeChild(*input, 0, ctx.Expr.NewCallable(input->Head().Pos(), "AsStruct", std::move(updatedChildren)));
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ resultType = ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TVariantExprType>(ctx.Expr.MakeType<TStructExprType>(itemTypes)));
+ }
+ else {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected Tuple or Struct type, but got: " << *inputType));
- return IGraphTransformer::TStatus::Error;
- }
-
- input->SetTypeAnn(resultType);
- return IGraphTransformer::TStatus::Ok;
- }
-
- IGraphTransformer::TStatus DemuxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- Y_UNUSED(output);
- if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ input->SetTypeAnn(resultType);
+ return IGraphTransformer::TStatus::Ok;
+ }
+
+ IGraphTransformer::TStatus DemuxWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ Y_UNUSED(output);
+ if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!EnsureListType(input->Head(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
auto listItemType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
if (!EnsureVariantType(input->Head().Pos(), *listItemType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- auto variantType = listItemType->Cast<TVariantExprType>();
- const TTypeAnnotationNode* resultType = nullptr;
- if (variantType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
- const TTupleExprType* tupleType = variantType->GetUnderlyingType()->Cast<TTupleExprType>();
- TTypeAnnotationNode::TListType listTypes;
- for (size_t i = 0; i < tupleType->GetSize(); ++i) {
- listTypes.push_back(ctx.Expr.MakeType<TListExprType>(tupleType->GetItems()[i]));
- }
- resultType = ctx.Expr.MakeType<TTupleExprType>(listTypes);
- }
- else {
- const TStructExprType* structType = variantType->GetUnderlyingType()->Cast<TStructExprType>();
- TVector<const TItemExprType*> listTypes;
- for (size_t i = 0; i < structType->GetSize(); ++i) {
- listTypes.push_back(
- ctx.Expr.MakeType<TItemExprType>(
- structType->GetItems()[i]->GetName(),
- ctx.Expr.MakeType<TListExprType>(structType->GetItems()[i]->GetItemType())
- )
- );
- }
- resultType = ctx.Expr.MakeType<TStructExprType>(listTypes);
- }
-
- input->SetTypeAnn(resultType);
- return IGraphTransformer::TStatus::Ok;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ auto variantType = listItemType->Cast<TVariantExprType>();
+ const TTypeAnnotationNode* resultType = nullptr;
+ if (variantType->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
+ const TTupleExprType* tupleType = variantType->GetUnderlyingType()->Cast<TTupleExprType>();
+ TTypeAnnotationNode::TListType listTypes;
+ for (size_t i = 0; i < tupleType->GetSize(); ++i) {
+ listTypes.push_back(ctx.Expr.MakeType<TListExprType>(tupleType->GetItems()[i]));
+ }
+ resultType = ctx.Expr.MakeType<TTupleExprType>(listTypes);
+ }
+ else {
+ const TStructExprType* structType = variantType->GetUnderlyingType()->Cast<TStructExprType>();
+ TVector<const TItemExprType*> listTypes;
+ for (size_t i = 0; i < structType->GetSize(); ++i) {
+ listTypes.push_back(
+ ctx.Expr.MakeType<TItemExprType>(
+ structType->GetItems()[i]->GetName(),
+ ctx.Expr.MakeType<TListExprType>(structType->GetItems()[i]->GetItemType())
+ )
+ );
+ }
+ resultType = ctx.Expr.MakeType<TStructExprType>(listTypes);
+ }
+
+ input->SetTypeAnn(resultType);
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus TimezoneIdWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
@@ -11765,7 +11765,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(optUi16Type);
+ input->SetTypeAnn(optUi16Type);
return IGraphTransformer::TStatus::Ok;
}
@@ -11796,7 +11796,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return convertStatus;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)));
return IGraphTransformer::TStatus::Ok;
}
@@ -11836,8 +11836,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return convertStatus;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(
- WithTzDate(dataType1->GetSlot()))));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(
+ WithTzDate(dataType1->GetSlot()))));
return IGraphTransformer::TStatus::Ok;
}
@@ -11859,9 +11859,9 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(WithoutTzDate(dataType1->GetSlot())));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(WithoutTzDate(dataType1->GetSlot())));
if (isOptional1) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -11974,7 +11974,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
if (!EnsureOptionalType(onErrorArg, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
-
+
const auto optionalTypeAnn = onErrorArg.GetTypeAnn();
if (!optionalTypeAnn) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(onErrorArg.Pos()), "Expected optional Bool, but got lambda"));
@@ -12683,39 +12683,39 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus AssumeAllMembersNullableAtOnceWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- Y_UNUSED(output);
- if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!EnsureStructOrOptionalStructType(input->Head(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- const TTypeAnnotationNode* itemType = input->Head().GetTypeAnn();
- if (itemType->GetKind() == ETypeAnnotationKind::Optional) {
- itemType = itemType->Cast<TOptionalExprType>()->GetItemType();
- }
-
- for (const auto& x : itemType->Cast<TStructExprType>()->GetItems()) {
- if (x->GetItemType()->GetKind() != ETypeAnnotationKind::Optional) {
- ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected all columns to be optional. Non optional column: " << x->GetName()));
- return IGraphTransformer::TStatus::Error;
- }
- }
-
- input->SetTypeAnn(input->Head().GetTypeAnn());
- return IGraphTransformer::TStatus::Ok;
- }
-
+ IGraphTransformer::TStatus AssumeAllMembersNullableAtOnceWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ Y_UNUSED(output);
+ if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!EnsureStructOrOptionalStructType(input->Head(), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ const TTypeAnnotationNode* itemType = input->Head().GetTypeAnn();
+ if (itemType->GetKind() == ETypeAnnotationKind::Optional) {
+ itemType = itemType->Cast<TOptionalExprType>()->GetItemType();
+ }
+
+ for (const auto& x : itemType->Cast<TStructExprType>()->GetItems()) {
+ if (x->GetItemType()->GetKind() != ETypeAnnotationKind::Optional) {
+ ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Head().Pos()), TStringBuilder() << "Expected all columns to be optional. Non optional column: " << x->GetName()));
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
+ input->SetTypeAnn(input->Head().GetTypeAnn());
+ return IGraphTransformer::TStatus::Ok;
+ }
+
TSyncFunctionsMap::TSyncFunctionsMap() {
Functions["Data"] = &DataWrapper;
Functions["DataSource"] = &DataSourceWrapper;
Functions["Key"] = &KeyWrapper;
- Functions[LeftName] = &LeftWrapper;
- Functions[RightName] = &RightWrapper;
- Functions[ConsName] = &ConsWrapper;
+ Functions[LeftName] = &LeftWrapper;
+ Functions[RightName] = &RightWrapper;
+ Functions[ConsName] = &ConsWrapper;
Functions["DataSink"] = &DataSinkWrapper;
Functions["Filter"] = &FilterWrapper;
Functions["OrderedFilter"] = &FilterWrapper;
@@ -12787,7 +12787,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["RotLeft"] = &ShiftWrapper;
Functions["ShiftRight"] = &ShiftWrapper;
Functions["RotRight"] = &ShiftWrapper;
- Functions[SyncName] = &SyncWrapper;
+ Functions[SyncName] = &SyncWrapper;
Functions["Concat"] = &ConcatWrapper;
Functions["AggrConcat"] = &AggrConcatWrapper;
Functions["Substring"] = &SubstringWrapper;
@@ -12823,7 +12823,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["RemoveMember"] = &RemoveMemberWrapper<false>;
Functions["ForceRemoveMember"] = &RemoveMemberWrapper<true>;
Functions["ReplaceMember"] = &ReplaceMemberWrapper;
- Functions["RemovePrefixMembers"] = &RemovePrefixMembersWrapper;
+ Functions["RemovePrefixMembers"] = &RemovePrefixMembersWrapper;
Functions["RemoveSystemMembers"] = &RemoveSystemMembersWrapper;
Functions["FlatMap"] = &FlatMapWrapper<false>;
Functions["OrderedFlatMap"] = &FlatMapWrapper<false>;
@@ -12853,24 +12853,24 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["Prepend"] = &PrependWrapper;
Functions["Extend"] = &ExtendWrapper;
Functions["OrderedExtend"] = &ExtendWrapper;
- Functions["Merge"] = &ExtendWrapper;
+ Functions["Merge"] = &ExtendWrapper;
Functions["Extract"] = &ExtractWrapper;
Functions["OrderedExtract"] = &ExtractWrapper;
Functions["UnionAll"] = &UnionAllWrapper;
- Functions["UnionMerge"] = &UnionAllWrapper;
+ Functions["UnionMerge"] = &UnionAllWrapper;
Functions["ListExtend"] = &ListExtendWrapper<false>;
Functions["ListExtendStrict"] = &ListExtendWrapper<true>;
Functions["ListUnionAll"] = &ListUnionAllWrapper;
Functions["ListZip"] = &ListZipWrapper;
Functions["ListZipAll"] = &ListZipAllWrapper;
Functions["Sort"] = &SortWrapper;
- Functions["AssumeSorted"] = &SortWrapper;
- Functions["AssumeUnique"] = &AssumeUniqueWrapper;
- Functions["AssumeAllMembersNullableAtOnce"] = &AssumeAllMembersNullableAtOnceWrapper;
+ Functions["AssumeSorted"] = &SortWrapper;
+ Functions["AssumeUnique"] = &AssumeUniqueWrapper;
+ Functions["AssumeAllMembersNullableAtOnce"] = &AssumeAllMembersNullableAtOnceWrapper;
Functions["Top"] = &TopWrapper;
Functions["TopSort"] = &TopWrapper;
Functions["KeepTop"] = &KeepTopWrapper;
- Functions["Unordered"] = &UnorderedWrapper;
+ Functions["Unordered"] = &UnorderedWrapper;
Functions["UnorderedSubquery"] = &UnorderedWrapper;
Functions["SortTraits"] = &SortTraitsWrapper;
Functions["SessionWindowTraits"] = &SessionWindowTraitsWrapper;
@@ -12904,7 +12904,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["PartitionByKey"] = &PartitionByKeyWrapper;
Functions["PartitionsByKeys"] = &PartitionsByKeysWrapper;
Functions["Reverse"] = &ReverseWrapper;
- Functions["Skip"] = &TakeWrapper;
+ Functions["Skip"] = &TakeWrapper;
Functions["Take"] = &TakeWrapper;
Functions["Limit"] = &TakeWrapper;
Functions["Fold"] = &FoldWrapper;
@@ -12938,7 +12938,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["Enum"] = &EnumWrapper;
Functions["AsVariant"] = &AsVariantWrapper;
Functions["AsEnum"] = &AsEnumWrapper;
- Functions["Contains"] = &ContainsLookupWrapper<true>;
+ Functions["Contains"] = &ContainsLookupWrapper<true>;
Functions["SqlIn"] = &SqlInWrapper;
Functions["Lookup"] = &ContainsLookupWrapper<false>;
Functions["DictItems"] = &DictItemsWrapper<EDictItems::Both>;
@@ -12953,7 +12953,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["DictFromKeys"] = &DictFromKeysWrapper;
Functions["If"] = &IfWrapper<false>;
Functions["IfStrict"] = &IfWrapper<true>;
- Functions[IfName] = &IfWorldWrapper;
+ Functions[IfName] = &IfWorldWrapper;
Functions[ForName] = &ForWorldWrapper;
Functions["IfPresent"] = &IfPresentWrapper;
Functions["StaticMap"] = &StaticMapWrapper;
@@ -12970,7 +12970,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["EnsurePersistable"] = &PersistableAssertWrapper;
Functions["PersistableRepr"] = &PersistableReprWrapper;
Functions["EnsureConvertibleTo"] = &TypeAssertWrapper<false>;
- Functions["EnsureTupleSize"] = &TupleSizeAssertWrapper;
+ Functions["EnsureTupleSize"] = &TupleSizeAssertWrapper;
Functions["Ensure"] = &EnsureWrapper;
Functions["TryMember"] = &TryMemberWrapper;
Functions["ToIndexDict"] = &ToIndexDictWrapper;
@@ -12992,8 +12992,8 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["JoinDict"] = &JoinDictWrapper;
Functions["MapJoinCore"] = &MapJoinCoreWrapper;
Functions["CommonJoinCore"] = &CommonJoinCoreWrapper;
- Functions["CombineCore"] = &CombineCoreWrapper;
- Functions["GroupingCore"] = &GroupingCoreWrapper;
+ Functions["CombineCore"] = &CombineCoreWrapper;
+ Functions["GroupingCore"] = &GroupingCoreWrapper;
Functions["HoppingTraits"] = &HoppingTraitsWrapper;
Functions["HoppingCore"] = &HoppingCoreWrapper;
Functions["MultiHoppingCore"] = &MultiHoppingCoreWrapper;
@@ -13036,7 +13036,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["DenseRank"] = &WinRankWrapper;
Functions["Ascending"] = &PresortWrapper;
Functions["Descending"] = &PresortWrapper;
- Functions["IsKeySwitch"] = &IsKeySwitchWrapper;
+ Functions["IsKeySwitch"] = &IsKeySwitchWrapper;
Functions["TableName"] = &TableNameWrapper;
Functions["FilterNullMembers"] = &FilterNullMembersWrapper;
Functions["SkipNullMembers"] = &SkipNullMembersWrapper;
@@ -13048,14 +13048,14 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["FlattenMembersType"] = &TypeArgWrapper<ETypeArgument::FlattenMembers>;
Functions["VariantUnderlyingType"] = &TypeArgWrapper<ETypeArgument::VariantUnderlying>;
Functions["Guess"] = &GuessWrapper;
- Functions["VariantItem"] = &VariantItemWrapper;
+ Functions["VariantItem"] = &VariantItemWrapper;
Functions["Visit"] = &VisitWrapper;
Functions["Way"] = &WayWrapper;
Functions["SqlAccess"] = &SqlAccessWrapper;
- Functions["SqlProcess"] = &SqlProcessWrapper;
- Functions["SqlReduce"] = &SqlReduceWrapper;
+ Functions["SqlProcess"] = &SqlProcessWrapper;
+ Functions["SqlReduce"] = &SqlReduceWrapper;
Functions["SqlExternalFunction"] = &SqlExternalFunctionWrapper;
- Functions["SqlExtractKey"] = &SqlExtractKeyWrapper;
+ Functions["SqlExtractKey"] = &SqlExtractKeyWrapper;
Functions["SqlReduceUdf"] = &SqlReduceUdfWrapper;
Functions["SqlProject"] = &SqlProjectWrapper;
Functions["SqlTypeFromYson"] = &SqlTypeFromYsonWrapper;
@@ -13076,7 +13076,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["PgGroup"] = &PgWhereWrapper;
Functions["PgWindow"] = &PgWindowWrapper;
Functions["PgAnonWindow"] = &PgAnonWindowWrapper;
- Functions["AutoDemuxList"] = &AutoDemuxListWrapper;
+ Functions["AutoDemuxList"] = &AutoDemuxListWrapper;
Functions["AggrCountInit"] = &AggrCountInitWrapper;
Functions["AggrCountUpdate"] = &AggrCountUpdateWrapper;
Functions["QueueCreate"] = &QueueCreateWrapper;
@@ -13086,11 +13086,11 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
Functions["Parameter"] = &ParameterWrapper;
Functions["WeakField"] = &WeakFieldWrapper;
Functions["TryWeakMemberFromDict"] = &TryWeakMemberFromDictWrapper;
- Functions["ByteString"] = &ByteStringWrapper;
- Functions["Utf8String"] = &Utf8StringWrapper;
- Functions["FromYsonSimpleType"] = &FromYsonSimpleType;
- Functions["Mux"] = &MuxWrapper;
- Functions["Demux"] = &DemuxWrapper;
+ Functions["ByteString"] = &ByteStringWrapper;
+ Functions["Utf8String"] = &Utf8StringWrapper;
+ Functions["FromYsonSimpleType"] = &FromYsonSimpleType;
+ Functions["Mux"] = &MuxWrapper;
+ Functions["Demux"] = &DemuxWrapper;
Functions["TimezoneId"] = &TimezoneIdWrapper;
Functions["TimezoneName"] = &TimezoneNameWrapper;
Functions["AddTimezone"] = &AddTimezoneWrapper;
@@ -13204,15 +13204,15 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
ExtFunctions["PgSelect"] = &PgSelectWrapper;
ExtFunctions["PgSetItem"] = &PgSetItemWrapper;
- ExtFunctions["TablePath"] = &TablePathWrapper;
- ExtFunctions["TableRecord"] = &TableRecordWrapper;
- ExtFunctions["Random"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Double>;
- ExtFunctions["RandomNumber"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Uint64>;
- ExtFunctions["RandomUuid"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Uuid>;
- ExtFunctions["Now"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Uint64>;
- ExtFunctions["CurrentUtcDate"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Date>;
- ExtFunctions["CurrentUtcDatetime"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Datetime>;
- ExtFunctions["CurrentUtcTimestamp"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Timestamp>;
+ ExtFunctions["TablePath"] = &TablePathWrapper;
+ ExtFunctions["TableRecord"] = &TableRecordWrapper;
+ ExtFunctions["Random"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Double>;
+ ExtFunctions["RandomNumber"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Uint64>;
+ ExtFunctions["RandomUuid"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Uuid>;
+ ExtFunctions["Now"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Uint64>;
+ ExtFunctions["CurrentUtcDate"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Date>;
+ ExtFunctions["CurrentUtcDatetime"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Datetime>;
+ ExtFunctions["CurrentUtcTimestamp"] = &DataGeneratorWrapper<NKikimr::NUdf::EDataSlot::Timestamp>;
ExtFunctions["CurrentTzDate"] = &CurrentTzWrapper<NKikimr::NUdf::EDataSlot::TzDate>;
ExtFunctions["CurrentTzDatetime"] = &CurrentTzWrapper<NKikimr::NUdf::EDataSlot::TzDatetime>;
ExtFunctions["CurrentTzTimestamp"] = &CurrentTzWrapper<NKikimr::NUdf::EDataSlot::TzTimestamp>;
@@ -13304,20 +13304,20 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
bool foundFunc = false;
for (auto& datasource : Types.DataSources) {
- if (!datasource->CanParse(*input)) {
+ if (!datasource->CanParse(*input)) {
continue;
}
foundFunc = true;
- status = DetermineIntents(*datasource, input, output, ctx);
+ status = DetermineIntents(*datasource, input, output, ctx);
break;
}
if (!foundFunc) {
for (auto& datasink : Types.DataSinks) {
- if (!datasink->CanParse(*input)) {
+ if (!datasink->CanParse(*input)) {
continue;
}
- status = DetermineIntents(*datasink, input, output, ctx);
+ status = DetermineIntents(*datasink, input, output, ctx);
break;
}
}
@@ -13345,13 +13345,13 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
const TTypeAnnotationContext& Types;
};
- class TExtCallableTypeAnnotationTransformer : public TCallableTransformerBase<TExtCallableTypeAnnotationTransformer> {
+ class TExtCallableTypeAnnotationTransformer : public TCallableTransformerBase<TExtCallableTypeAnnotationTransformer> {
public:
TExtCallableTypeAnnotationTransformer(TTypeAnnotationContext& types, bool instantOnly)
- : TCallableTransformerBase<TExtCallableTypeAnnotationTransformer>(types, instantOnly)
+ : TCallableTransformerBase<TExtCallableTypeAnnotationTransformer>(types, instantOnly)
{}
- TMaybe<IGraphTransformer::TStatus> ProcessCore(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ TMaybe<IGraphTransformer::TStatus> ProcessCore(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
auto& functions = TSyncFunctionsMap::Instance().Functions;
auto& extFunctions = TSyncFunctionsMap::Instance().ExtFunctions;
auto& columnOrderFunctions = TSyncFunctionsMap::Instance().ColumnOrderFunctions;
@@ -13364,7 +13364,7 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
TExtContext funcCtx(ctx, Types);
status = (*func)(input, output, funcCtx);
} else {
- return Nothing();
+ return Nothing();
}
if (status == IGraphTransformer::TStatus::Ok && Types.OrderedColumns && !Types.LookupColumnOrder(*input)) {
@@ -13381,49 +13381,49 @@ template <NKikimr::NUdf::EDataSlot DataSlot>
return {};
}
- IGraphTransformer::TStatus ProcessUnknown(const TExprNode::TPtr& input, TExprContext& ctx) {
+ IGraphTransformer::TStatus ProcessUnknown(const TExprNode::TPtr& input, TExprContext& ctx) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder()
<< "(Core type annotation) Unsupported function: " << input->Content()));
- return TStatus::Error;
+ return TStatus::Error;
}
- IGraphTransformer::TStatus ValidateProviderCommitResult(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (!input->GetTypeAnn() || input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) {
+ IGraphTransformer::TStatus ValidateProviderCommitResult(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (!input->GetTypeAnn() || input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad datasink commit result"));
- return TStatus::Error;
+ return TStatus::Error;
}
- return TStatus::Ok;
+ return TStatus::Ok;
}
- IGraphTransformer::TStatus ValidateProviderReadResult(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (!input->GetTypeAnn() ||
- input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tuple ||
- input->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() != 2 ||
- input->GetTypeAnn()->Cast<TTupleExprType>()->GetItems()[0]->GetKind() != ETypeAnnotationKind::World) {
+ IGraphTransformer::TStatus ValidateProviderReadResult(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (!input->GetTypeAnn() ||
+ input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tuple ||
+ input->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() != 2 ||
+ input->GetTypeAnn()->Cast<TTupleExprType>()->GetItems()[0]->GetKind() != ETypeAnnotationKind::World) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad datasource read result"));
- return TStatus::Error;
+ return TStatus::Error;
}
- return TStatus::Ok;
+ return TStatus::Ok;
}
- IGraphTransformer::TStatus ValidateProviderWriteResult(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (!input->GetTypeAnn() || input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) {
+ IGraphTransformer::TStatus ValidateProviderWriteResult(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (!input->GetTypeAnn() || input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad datasink write result"));
- return TStatus::Error;
+ return TStatus::Error;
}
- return TStatus::Ok;
+ return TStatus::Ok;
}
- IGraphTransformer::TStatus ValidateProviderConfigureResult(const TExprNode::TPtr& input, TExprContext& ctx) {
+ IGraphTransformer::TStatus ValidateProviderConfigureResult(const TExprNode::TPtr& input, TExprContext& ctx) {
if (!input->GetTypeAnn() || input->GetTypeAnn()->GetKind() != input->Head().GetTypeAnn()->GetKind()) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Bad provider configure result"));
- return TStatus::Error;
+ return TStatus::Error;
}
- return TStatus::Ok;
+ return TStatus::Ok;
}
- IGraphTransformer& GetTransformer(IDataProvider& provider) const {
- return provider.GetTypeAnnotationTransformer(InstantOnly);
+ IGraphTransformer& GetTransformer(IDataProvider& provider) const {
+ return provider.GetTypeAnnotationTransformer(InstantOnly);
}
};
} // namespace NTypeAnnInpl
diff --git a/ydb/library/yql/core/type_ann/type_ann_expr.cpp b/ydb/library/yql/core/type_ann/type_ann_expr.cpp
index f67d3b4120..9a027ce3bd 100644
--- a/ydb/library/yql/core/type_ann/type_ann_expr.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_expr.cpp
@@ -139,12 +139,12 @@ private:
case TExprNode::EState::TypeInProgress:
return IGraphTransformer::TStatus::Async;
case TExprNode::EState::TypePending:
- if (start->Type() == TExprNode::Lambda) {
+ if (start->Type() == TExprNode::Lambda) {
if (!start->Head().GetTypeAnn()) {
- return TStatus::Ok;
+ return TStatus::Ok;
} else if (start->Head().ChildrenSize() == 0) {
- break;
- }
+ break;
+ }
}
if (start->Type() == TExprNode::Arguments || start->Type() == TExprNode::Argument) {
@@ -153,9 +153,9 @@ private:
return TStatus(TStatus::Repeat, true);
case TExprNode::EState::TypeComplete:
- case TExprNode::EState::ConstrInProgress:
- case TExprNode::EState::ConstrPending:
- case TExprNode::EState::ConstrComplete:
+ case TExprNode::EState::ConstrInProgress:
+ case TExprNode::EState::ConstrPending:
+ case TExprNode::EState::ConstrComplete:
case TExprNode::EState::ExecutionInProgress:
case TExprNode::EState::ExecutionRequired:
case TExprNode::EState::ExecutionPending:
@@ -233,9 +233,9 @@ private:
case TExprNode::EState::TypePending:
break;
case TExprNode::EState::TypeComplete:
- case TExprNode::EState::ConstrInProgress:
- case TExprNode::EState::ConstrPending:
- case TExprNode::EState::ConstrComplete:
+ case TExprNode::EState::ConstrInProgress:
+ case TExprNode::EState::ConstrPending:
+ case TExprNode::EState::ConstrComplete:
case TExprNode::EState::ExecutionInProgress:
case TExprNode::EState::ExecutionRequired:
case TExprNode::EState::ExecutionPending:
@@ -251,7 +251,7 @@ private:
switch (input->Type()) {
case TExprNode::Atom:
{
- input->SetTypeAnn(ctx.MakeType<TUnitExprType>());
+ input->SetTypeAnn(ctx.MakeType<TUnitExprType>());
CheckExpected(*input, ctx);
return TStatus::Ok;
}
@@ -293,15 +293,15 @@ private:
return TStatus::Error;
}
- children.push_back(child->GetTypeAnn());
- if (child->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Unit) {
+ children.push_back(child->GetTypeAnn());
+ if (child->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Unit) {
isUnit = true;
}
}
- input->SetTypeAnn(isUnit ?
+ input->SetTypeAnn(isUnit ?
(const TTypeAnnotationNode*)ctx.MakeType<TUnitExprType>() :
- ctx.MakeType<TTupleExprType>(children));
+ ctx.MakeType<TTupleExprType>(children));
CheckExpected(*input, ctx);
return TStatus::Ok;
}
@@ -363,7 +363,7 @@ private:
input->SetTypeAnn(input->Tail().GetTypeAnn());
}
- if (input->GetTypeAnn()) {
+ if (input->GetTypeAnn()) {
CheckExpected(*input, ctx);
}
@@ -371,7 +371,7 @@ private:
}
case TExprNode::Argument:
- if (input->GetTypeAnn()) {
+ if (input->GetTypeAnn()) {
if (input->Type() == TExprNode::Lambda) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Unable to use lambda as argument"));
input->SetState(TExprNode::EState::Error);
@@ -430,7 +430,7 @@ private:
}
if (status == TStatus::Ok) {
- if (!input->GetTypeAnn()) {
+ if (!input->GetTypeAnn()) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Node is not annotated yet"));
input->SetState(TExprNode::EState::Error);
return TStatus::Error;
@@ -458,7 +458,7 @@ private:
case TExprNode::World:
{
- input->SetTypeAnn(ctx.MakeType<TWorldExprType>());
+ input->SetTypeAnn(ctx.MakeType<TWorldExprType>());
CheckExpected(*input, ctx);
return TStatus::Ok;
}
@@ -466,7 +466,7 @@ private:
case TExprNode::Arguments:
{
if (input->Children().empty()) {
- if (input->GetTypeAnn()) {
+ if (input->GetTypeAnn()) {
input->SetState(TExprNode::EState::TypeComplete);
return TStatus::Ok;
}
@@ -491,7 +491,7 @@ private:
return combinedStatus;
}
- input->SetTypeAnn(ctx.MakeType<TUnitExprType>());
+ input->SetTypeAnn(ctx.MakeType<TUnitExprType>());
return TStatus::Ok;
}
@@ -517,7 +517,7 @@ private:
Y_UNUSED(ctx);
auto it = Types.ExpectedTypes.find(input.UniqueId());
if (it != Types.ExpectedTypes.end()) {
- YQL_ENSURE(IsSameAnnotation(*input.GetTypeAnn(), *it->second),
+ YQL_ENSURE(IsSameAnnotation(*input.GetTypeAnn(), *it->second),
"Rewrite error, type should be : " <<
*it->second << ", but it is: " << *input.GetTypeAnn() << " for node " << input.Content());
}
@@ -556,7 +556,7 @@ private:
IGraphTransformer::TStatus CheckWholeProgramType(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
output = input;
- if (input->Type() == TExprNode::Lambda || input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) {
+ if (input->Type() == TExprNode::Lambda || input->GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), "Return must be world"));
input->SetState(TExprNode::EState::Error);
return IGraphTransformer::TStatus::Error;
@@ -575,7 +575,7 @@ TAutoPtr<IGraphTransformer> CreateFullTypeAnnotationTransformer(
TTypeAnnotationContext& typeAnnotationContext)
{
TVector<TTransformStage> transformers;
- auto issueCode = TIssuesIds::CORE_PRE_TYPE_ANN;
+ auto issueCode = TIssuesIds::CORE_PRE_TYPE_ANN;
transformers.push_back(TTransformStage(
CreateFunctorTransformer(&ExpandApply),
"ExpandApply",
@@ -596,13 +596,13 @@ TAutoPtr<IGraphTransformer> CreateFullTypeAnnotationTransformer(
CreateIODiscoveryTransformer(typeAnnotationContext),
"IODiscovery",
issueCode));
- transformers.push_back(TTransformStage(
- CreateEpochsTransformer(typeAnnotationContext),
- "Epochs",
- issueCode));
+ transformers.push_back(TTransformStage(
+ CreateEpochsTransformer(typeAnnotationContext),
+ "Epochs",
+ issueCode));
transformers.push_back(TTransformStage(
- CreateIntentDeterminationTransformer(typeAnnotationContext),
+ CreateIntentDeterminationTransformer(typeAnnotationContext),
"IntentDetermination",
issueCode));
transformers.push_back(TTransformStage(
@@ -618,9 +618,9 @@ TAutoPtr<IGraphTransformer> CreateFullTypeAnnotationTransformer(
"RewriteIO",
issueCode));
- issueCode = TIssuesIds::CORE_TYPE_ANN;
- auto callableTransformer = CreateExtCallableTypeAnnotationTransformer(typeAnnotationContext);
- auto typeTransformer = CreateTypeAnnotationTransformer(callableTransformer, typeAnnotationContext);
+ issueCode = TIssuesIds::CORE_TYPE_ANN;
+ auto callableTransformer = CreateExtCallableTypeAnnotationTransformer(typeAnnotationContext);
+ auto typeTransformer = CreateTypeAnnotationTransformer(callableTransformer, typeAnnotationContext);
transformers.push_back(TTransformStage(
typeTransformer,
"TypeAnnotation",
@@ -663,7 +663,7 @@ TExprNode::TPtr ParseAndAnnotate(
}
TExprNode::TPtr exprRoot;
- if (!CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr)) {
+ if (!CompileExpr(*astRes.Root, exprRoot, exprCtx, nullptr)) {
return nullptr;
}
diff --git a/ydb/library/yql/core/type_ann/type_ann_expr.h b/ydb/library/yql/core/type_ann/type_ann_expr.h
index 914e71cede..894612fffa 100644
--- a/ydb/library/yql/core/type_ann/type_ann_expr.h
+++ b/ydb/library/yql/core/type_ann/type_ann_expr.h
@@ -11,7 +11,7 @@ namespace NYql {
TAutoPtr<IGraphTransformer> CreateTypeAnnotationTransformer(
TAutoPtr<IGraphTransformer> callableTransformer, TTypeAnnotationContext& types);
-TAutoPtr<IGraphTransformer> CreateFullTypeAnnotationTransformer(bool instant, bool wholeProgram, TTypeAnnotationContext& types);
+TAutoPtr<IGraphTransformer> CreateFullTypeAnnotationTransformer(bool instant, bool wholeProgram, TTypeAnnotationContext& types);
bool SyncAnnotateTypes(
TExprNode::TPtr& root, TExprContext& ctx, bool wholeProgram,
diff --git a/ydb/library/yql/core/type_ann/type_ann_impl.h b/ydb/library/yql/core/type_ann/type_ann_impl.h
index 3d940ebd73..d80f6cbf9e 100644
--- a/ydb/library/yql/core/type_ann/type_ann_impl.h
+++ b/ydb/library/yql/core/type_ann/type_ann_impl.h
@@ -30,8 +30,8 @@ namespace NTypeAnnImpl {
IGraphTransformer::TStatus MapJoinCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus CommonJoinCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus EquiJoinWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
- IGraphTransformer::TStatus CombineCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
- IGraphTransformer::TStatus GroupingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
+ IGraphTransformer::TStatus CombineCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
+ IGraphTransformer::TStatus GroupingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
TMaybe<ui32> FindOrReportMissingMember(TStringBuf memberName, TPositionHandle pos, const TStructExprType& structType, TContext& ctx);
diff --git a/ydb/library/yql/core/type_ann/type_ann_join.cpp b/ydb/library/yql/core/type_ann/type_ann_join.cpp
index 18516a8ab8..42ac49b9b8 100644
--- a/ydb/library/yql/core/type_ann/type_ann_join.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_join.cpp
@@ -39,42 +39,42 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- const auto leftItemType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
- if (leftItemType->GetKind() == ETypeAnnotationKind::Struct) {
- auto structType = leftItemType->Cast<TStructExprType>();
- if (AnyOf(structType->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) {
- output = ctx.Expr.ChangeChild(*input, 0,
- ctx.Expr.Builder(input->Child(0)->Pos())
- .Callable("RemovePrefixMembers")
- .Add(0, input->ChildPtr(0))
- .List(1)
- .Atom(0, "_yql_sys_", TNodeFlags::Default)
- .Seal()
- .Seal()
- .Build()
- );
- return IGraphTransformer::TStatus::Repeat;
- }
- }
-
- const TTypeAnnotationNode* rightItemType = input->Child(1)->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
- if (rightItemType->GetKind() == ETypeAnnotationKind::Struct) {
- auto structType = rightItemType->Cast<TStructExprType>();
- if (AnyOf(structType->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) {
- output = ctx.Expr.ChangeChild(*input, 1,
- ctx.Expr.Builder(input->Child(1)->Pos())
- .Callable("RemovePrefixMembers")
- .Add(0, input->ChildPtr(1))
- .List(1)
- .Atom(0, "_yql_sys_", TNodeFlags::Default)
- .Seal()
- .Seal()
- .Build()
- );
- return IGraphTransformer::TStatus::Repeat;
- }
- }
-
+ const auto leftItemType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ if (leftItemType->GetKind() == ETypeAnnotationKind::Struct) {
+ auto structType = leftItemType->Cast<TStructExprType>();
+ if (AnyOf(structType->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) {
+ output = ctx.Expr.ChangeChild(*input, 0,
+ ctx.Expr.Builder(input->Child(0)->Pos())
+ .Callable("RemovePrefixMembers")
+ .Add(0, input->ChildPtr(0))
+ .List(1)
+ .Atom(0, "_yql_sys_", TNodeFlags::Default)
+ .Seal()
+ .Seal()
+ .Build()
+ );
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ }
+
+ const TTypeAnnotationNode* rightItemType = input->Child(1)->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ if (rightItemType->GetKind() == ETypeAnnotationKind::Struct) {
+ auto structType = rightItemType->Cast<TStructExprType>();
+ if (AnyOf(structType->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) {
+ output = ctx.Expr.ChangeChild(*input, 1,
+ ctx.Expr.Builder(input->Child(1)->Pos())
+ .Callable("RemovePrefixMembers")
+ .Add(0, input->ChildPtr(1))
+ .List(1)
+ .Atom(0, "_yql_sys_", TNodeFlags::Default)
+ .Seal()
+ .Seal()
+ .Build()
+ );
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ }
+
auto& lambda1 = input->ChildRef(2);
if (!UpdateLambdaAllArgumentsTypes(lambda1, {leftItemType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
@@ -85,7 +85,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda1->GetTypeAnn() || !lambda2->GetTypeAnn()) {
+ if (!lambda1->GetTypeAnn() || !lambda2->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -97,17 +97,17 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- if (!EnsureComparableKey(lambda1->Pos(), lambda1->GetTypeAnn(), ctx.Expr)) {
+ if (!EnsureComparableKey(lambda1->Pos(), lambda1->GetTypeAnn(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!EnsureComparableKey(lambda2->Pos(), lambda2->GetTypeAnn(), ctx.Expr)) {
+ if (!EnsureComparableKey(lambda2->Pos(), lambda2->GetTypeAnn(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!IsSameAnnotation(*lambda1->GetTypeAnn(), *lambda2->GetTypeAnn())) {
+ if (!IsSameAnnotation(*lambda1->GetTypeAnn(), *lambda2->GetTypeAnn())) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Pos()), TStringBuilder() << "mismatch of key extractors types, "
- << *lambda1->GetTypeAnn() << " != " << *lambda2->GetTypeAnn()));
+ << *lambda1->GetTypeAnn() << " != " << *lambda2->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -123,7 +123,7 @@ namespace NTypeAnnImpl {
}
auto tupleType = ctx.Expr.MakeType<TTupleExprType>(tupleItems);
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(tupleType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(tupleType));
return IGraphTransformer::TStatus::Ok;
}
@@ -243,12 +243,12 @@ namespace NTypeAnnImpl {
auto optionsNode = input->Child(input->ChildrenSize() - 1);
TJoinOptions options;
auto status = ValidateEquiJoinOptions(input->Pos(), *optionsNode, options, ctx.Expr);
- if (status != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
+ if (status != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
TJoinLabels labels;
- TExprNode::TListType updatedChildren;
+ TExprNode::TListType updatedChildren;
for (ui32 idx = 0; idx < numLists; ++idx) {
auto& listPair = *input->Child(idx);
if (!EnsureTupleSize(listPair, 2, ctx.Expr)) {
@@ -260,7 +260,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- const TTypeAnnotationNode* itemType = list.GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ const TTypeAnnotationNode* itemType = list.GetTypeAnn()->Cast<TListExprType>()->GetItemType();
if (itemType->GetKind() != ETypeAnnotationKind::Struct) {
ctx.Expr.AddError(TIssue(
ctx.Expr.GetPosition(list.Pos()),
@@ -271,21 +271,21 @@ namespace NTypeAnnImpl {
auto structType = itemType->Cast<TStructExprType>();
if (!options.KeepSysColumns && AnyOf(structType->GetItems(), [](const TItemExprType* structItem) { return structItem->GetName().StartsWith("_yql_sys_"); })) {
- if (updatedChildren.empty()) {
- updatedChildren = input->ChildrenList();
- }
- updatedChildren[idx] = ctx.Expr.ChangeChild(listPair, 0,
- ctx.Expr.Builder(list.Pos())
- .Callable("RemovePrefixMembers")
- .Add(0, listPair.HeadPtr())
- .List(1)
- .Atom(0, "_yql_sys_", TNodeFlags::Default)
- .Seal()
- .Seal()
- .Build()
- );
- continue;
- }
+ if (updatedChildren.empty()) {
+ updatedChildren = input->ChildrenList();
+ }
+ updatedChildren[idx] = ctx.Expr.ChangeChild(listPair, 0,
+ ctx.Expr.Builder(list.Pos())
+ .Callable("RemovePrefixMembers")
+ .Add(0, listPair.HeadPtr())
+ .List(1)
+ .Atom(0, "_yql_sys_", TNodeFlags::Default)
+ .Seal()
+ .Seal()
+ .Build()
+ );
+ continue;
+ }
if (auto err = labels.Add(ctx.Expr, *listPair.Child(1), structType)) {
ctx.Expr.AddError(*err);
ctx.Expr.AddError(TIssue(
@@ -295,10 +295,10 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
}
- if (!updatedChildren.empty()) {
- output = ctx.Expr.ChangeChildren(*input, std::move(updatedChildren));
- return IGraphTransformer::TStatus::Repeat;
- }
+ if (!updatedChildren.empty()) {
+ output = ctx.Expr.ChangeChildren(*input, std::move(updatedChildren));
+ return IGraphTransformer::TStatus::Repeat;
+ }
auto joins = input->Child(input->ChildrenSize() - 2);
const TStructExprType* resultType = nullptr;
@@ -307,7 +307,7 @@ namespace NTypeAnnImpl {
return status;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(resultType));
return IGraphTransformer::TStatus::Ok;
}
@@ -326,7 +326,7 @@ namespace NTypeAnnImpl {
template<class TLeftType>
IGraphTransformer::TStatus MapJoinCoreWrapperT(const TExprNode::TPtr& input, const TLeftType& leftItemType, TContext& ctx) {
constexpr bool ByStruct = std::is_same<TLeftType, TStructExprType>::value;
- const auto dictType = input->Child(1)->GetTypeAnn()->Cast<TDictExprType>();
+ const auto dictType = input->Child(1)->GetTypeAnn()->Cast<TDictExprType>();
const auto dictPayloadType = dictType->GetPayloadType();
if (!EnsureAtom(*input->Child(2), ctx.Expr)) {
@@ -694,21 +694,21 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
}
- else if (optionName == "memLimit") {
- if (!EnsureTupleSize(*child, 2, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!EnsureAtom(*child->Child(1), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ else if (optionName == "memLimit") {
+ if (!EnsureTupleSize(*child, 2, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!EnsureAtom(*child->Child(1), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (ui64 memLimit = 0ULL; !TryFromString(child->Child(1)->Content(), memLimit)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(child->Child(1)->Pos()), TStringBuilder() <<
- "Bad memLimit value: " << child->Child(1)->Content()));
- return IGraphTransformer::TStatus::Error;
- }
- }
+ "Bad memLimit value: " << child->Child(1)->Content()));
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
else if (optionName == "any") {
if (!EnsureTupleSize(*child, 2, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
diff --git a/ydb/library/yql/core/type_ann/type_ann_list.cpp b/ydb/library/yql/core/type_ann/type_ann_list.cpp
index b145cde389..4fff220fdc 100644
--- a/ydb/library/yql/core/type_ann/type_ann_list.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_list.cpp
@@ -14,8 +14,8 @@
namespace NYql {
namespace NTypeAnnImpl {
-using namespace NNodes;
-
+using namespace NNodes;
+
namespace {
bool IsEmptyList(const TExprNode::TPtr& x) {
return x->GetTypeAnn() && x->GetTypeAnn()->GetKind() == ETypeAnnotationKind::EmptyList;
@@ -665,7 +665,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -766,7 +766,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -775,7 +775,7 @@ namespace {
return IGraphTransformer::TStatus::Repeat;
}
- if (!EnsureComputableType(lambda->Pos(), *lambda->GetTypeAnn(), ctx.Expr)) {
+ if (!EnsureComputableType(lambda->Pos(), *lambda->GetTypeAnn(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -807,25 +807,25 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!EnsureSeqOrOptionalType(lambda->Pos(), *lambda->GetTypeAnn(), ctx.Expr)) {
+ if (!EnsureSeqOrOptionalType(lambda->Pos(), *lambda->GetTypeAnn(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- auto retKind = lambda->GetTypeAnn()->GetKind();
+ auto retKind = lambda->GetTypeAnn()->GetKind();
const TTypeAnnotationNode* retItemType;
if (retKind == ETypeAnnotationKind::List) {
- retItemType = lambda->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ retItemType = lambda->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
} else if (retKind == ETypeAnnotationKind::Optional) {
- retItemType = lambda->GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType();
+ retItemType = lambda->GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType();
} else {
- retItemType = lambda->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
+ retItemType = lambda->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(retItemType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(retItemType));
return IGraphTransformer::TStatus::Ok;
}
@@ -856,11 +856,11 @@ namespace {
return status;
}
- if (!UpdateLambdaAllArgumentsTypes(lambda, {itemType, initState.GetTypeAnn()}, ctx.Expr)) {
+ if (!UpdateLambdaAllArgumentsTypes(lambda, {itemType, initState.GetTypeAnn()}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- auto lambdaType = lambda->GetTypeAnn();
+ auto lambdaType = lambda->GetTypeAnn();
if (!lambdaType) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -878,10 +878,10 @@ namespace {
auto tupleExprType = lambdaType->Cast<TTupleExprType>();
auto updatedListType = tupleExprType->GetItems()[0];
auto stateType = tupleExprType->GetItems()[1];
- if (!IsSameAnnotation(*initState.GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*initState.GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() <<
"Mismatch of update lambda state and init state type: " <<
- *stateType << " != " << *initState.GetTypeAnn()));
+ *stateType << " != " << *initState.GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -917,7 +917,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- auto lambdaType = initLambda->GetTypeAnn();
+ auto lambdaType = initLambda->GetTypeAnn();
if (!lambdaType) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -936,7 +936,7 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(updateLambda, {itemType, stateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- auto updateLambdaType = updateLambda->GetTypeAnn();
+ auto updateLambdaType = updateLambda->GetTypeAnn();
if (!updateLambdaType) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -1035,7 +1035,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -1710,9 +1710,9 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64));
if (isOptional) {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(input->GetTypeAnn()));
}
return IGraphTransformer::TStatus::Ok;
@@ -1750,7 +1750,7 @@ namespace {
}
const TTypeAnnotationNode* itemType = type->Cast<TListExprType>()->GetItemType();
- input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(itemType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(itemType));
return IGraphTransformer::TStatus::Ok;
}
@@ -1769,7 +1769,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(type);
+ input->SetTypeAnn(type);
return IGraphTransformer::TStatus::Ok;
}
@@ -1784,7 +1784,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(itemType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(itemType));
return IGraphTransformer::TStatus::Ok;
}
@@ -1819,7 +1819,7 @@ namespace {
}
}
- input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(itemType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(itemType));
return IGraphTransformer::TStatus::Ok;
}
@@ -1894,7 +1894,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(itemType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(itemType));
return IGraphTransformer::TStatus::Ok;
}
@@ -1972,7 +1972,7 @@ namespace {
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(commonType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(commonType));
return IGraphTransformer::TStatus::Ok;
}
@@ -2103,7 +2103,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -2221,7 +2221,7 @@ namespace {
}
}
- input->SetTypeAnn(listType);
+ input->SetTypeAnn(listType);
return IGraphTransformer::TStatus::Ok;
}
@@ -2243,11 +2243,11 @@ namespace {
auto tmpArg2 = ctx.Expr.NewArgument(input->Pos(), "tmp2");
THashMap<TStringBuf, std::pair<const TTypeAnnotationNode*, ui32>> members;
ui32 inputsCount = input->ChildrenSize();
- auto checkStructType = [&members, &ctx, tmpArg1, tmpArg2, inputsCount, pos = input->Pos()](TExprNode& input) -> const TStructExprType* {
+ auto checkStructType = [&members, &ctx, tmpArg1, tmpArg2, inputsCount, pos = input->Pos()](TExprNode& input) -> const TStructExprType* {
if (!EnsureListType(input, ctx.Expr)) {
return nullptr;
}
- auto itemType = input.GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ auto itemType = input.GetTypeAnn()->Cast<TListExprType>()->GetItemType();
if (!EnsureStructType(input.Pos(), *itemType, ctx.Expr)) {
return nullptr;
}
@@ -2330,7 +2330,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(structType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(structType));
return IGraphTransformer::TStatus::Ok;
}
@@ -2561,14 +2561,14 @@ namespace {
bool ValidateSortDirections(TExprNode& direction, TExprContext& ctx, bool& isTuple) {
bool isOkAscending = false;
- if (direction.GetTypeAnn()) {
- if (direction.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Data &&
- direction.GetTypeAnn()->Cast<TDataExprType>()->GetSlot() == EDataSlot::Bool) {
+ if (direction.GetTypeAnn()) {
+ if (direction.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Data &&
+ direction.GetTypeAnn()->Cast<TDataExprType>()->GetSlot() == EDataSlot::Bool) {
isOkAscending = true;
}
- else if (direction.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
+ else if (direction.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
isTuple = true;
- auto& items = direction.GetTypeAnn()->Cast<TTupleExprType>()->GetItems();
+ auto& items = direction.GetTypeAnn()->Cast<TTupleExprType>()->GetItems();
isOkAscending = true;
for (auto& child : items) {
if (child->GetKind() != ETypeAnnotationKind::Data || child->Cast<TDataExprType>()->GetSlot() != EDataSlot::Bool) {
@@ -2602,19 +2602,19 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
if (isTuple) {
- const auto directionTupleSize = direction->GetTypeAnn()->Cast<TTupleExprType>()->GetSize();
+ const auto directionTupleSize = direction->GetTypeAnn()->Cast<TTupleExprType>()->GetSize();
if (directionTupleSize == 0 || directionTupleSize >= 2) {
- if (lambda->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tuple) {
+ if (lambda->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tuple) {
ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << "expected tuple type as lambda result, but got: " << *lambda->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
- const auto lambdaTupleSize = lambda->GetTypeAnn()->Cast<TTupleExprType>()->GetSize();
+ const auto lambdaTupleSize = lambda->GetTypeAnn()->Cast<TTupleExprType>()->GetSize();
if (lambdaTupleSize != directionTupleSize) {
ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() <<
"Mismatch of lambda result and ascending parameters sizes: " <<
@@ -2624,7 +2624,7 @@ namespace {
}
}
- if (!lambda->GetTypeAnn()->IsComparable()) {
+ if (!lambda->GetTypeAnn()->IsComparable()) {
ctx.AddError(TIssue(ctx.GetPosition(lambda->Pos()), TStringBuilder() << "Expected comparable type, but got: " << *lambda->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -2723,29 +2723,29 @@ namespace {
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus UnorderedWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ IGraphTransformer::TStatus UnorderedWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (input->Head().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple) {
output = input->HeadPtr();
- return IGraphTransformer::TStatus::Repeat;
- }
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
if (IsEmptyList(input->Head())) {
output = input->HeadPtr();
return IGraphTransformer::TStatus::Repeat;
}
if (!EnsureNewSeqType<false>(input->Head(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
input->SetTypeAnn(input->Head().GetTypeAnn());
- return IGraphTransformer::TStatus::Ok;
- }
-
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus SortTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
if (!EnsureArgsCount(*input, 3, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
@@ -2755,7 +2755,7 @@ namespace {
}
const auto& listTypeNode = input->Head();
- auto& listType = *listTypeNode.GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto& listType = *listTypeNode.GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (listType.GetKind() == ETypeAnnotationKind::EmptyList) {
output = ctx.Expr.NewCallable(input->Pos(), "Void", {});
return IGraphTransformer::TStatus::Repeat;
@@ -2948,7 +2948,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
+ input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
return IGraphTransformer::TStatus::Ok;
}
@@ -2974,7 +2974,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaKeySelector->GetTypeAnn()) {
+ if (!lambdaKeySelector->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -2982,8 +2982,8 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (lambdaKeySelector->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple &&
- lambdaKeySelector->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() < 2) {
+ if (lambdaKeySelector->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple &&
+ lambdaKeySelector->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() < 2) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaKeySelector->Pos()), "Tuple must contain at least 2 items"));
return IGraphTransformer::TStatus::Error;
}
@@ -2993,26 +2993,26 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaListHandler->GetTypeAnn()) {
+ if (!lambdaListHandler->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto retKind = lambdaListHandler->GetTypeAnn()->GetKind();
+ auto retKind = lambdaListHandler->GetTypeAnn()->GetKind();
if (retKind != ETypeAnnotationKind::List && retKind != ETypeAnnotationKind::Optional
&& retKind != ETypeAnnotationKind::Stream) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaListHandler->Pos()), TStringBuilder() <<
- "Expected either list, stream or optional, but got: " << *lambdaListHandler->GetTypeAnn()));
+ "Expected either list, stream or optional, but got: " << *lambdaListHandler->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
if (retKind == ETypeAnnotationKind::List) {
- input->SetTypeAnn(lambdaListHandler->GetTypeAnn());
+ input->SetTypeAnn(lambdaListHandler->GetTypeAnn());
} else if (retKind == ETypeAnnotationKind::Optional) {
- auto itemType = lambdaListHandler->GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType();
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(itemType));
+ auto itemType = lambdaListHandler->GetTypeAnn()->Cast<TOptionalExprType>()->GetItemType();
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(itemType));
} else {
- auto itemType = lambdaListHandler->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(itemType));
+ auto itemType = lambdaListHandler->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(itemType));
}
return IGraphTransformer::TStatus::Ok;
@@ -3058,19 +3058,19 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaKeySelector->GetTypeAnn()) {
+ if (!lambdaKeySelector->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!lambdaKeySelector->GetTypeAnn()->IsHashable() || !lambdaKeySelector->GetTypeAnn()->IsEquatable()) {
+ if (!lambdaKeySelector->GetTypeAnn()->IsHashable() || !lambdaKeySelector->GetTypeAnn()->IsEquatable()) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaKeySelector->Pos()), TStringBuilder()
- << "Expected hashable and equatable type for key, but got: " << *lambdaKeySelector->GetTypeAnn()));
+ << "Expected hashable and equatable type for key, but got: " << *lambdaKeySelector->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
auto& lambdaListHandler = input->ChildRef(4);
TTypeAnnotationNode::TListType tupleItems = {
- lambdaKeySelector->GetTypeAnn(),
+ lambdaKeySelector->GetTypeAnn(),
ctx.Expr.MakeType<TStreamExprType>(itemType)
};
@@ -3080,11 +3080,11 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaListHandler->GetTypeAnn()) {
+ if (!lambdaListHandler->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!EnsureSeqOrOptionalType(lambdaListHandler->Pos(), *lambdaListHandler->GetTypeAnn(), ctx.Expr)) {
+ if (!EnsureSeqOrOptionalType(lambdaListHandler->Pos(), *lambdaListHandler->GetTypeAnn(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -3093,7 +3093,7 @@ namespace {
retKind != ETypeAnnotationKind::Stream)
{
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaListHandler->Pos()), TStringBuilder() <<
- "Expected either list, stream or optional, but got: " << *lambdaListHandler->GetTypeAnn()));
+ "Expected either list, stream or optional, but got: " << *lambdaListHandler->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -3263,21 +3263,21 @@ namespace {
auto& lambda = input->TailRef();
const auto itemType = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
- if (!UpdateLambdaAllArgumentsTypes(lambda, {itemType, input->Child(1)->GetTypeAnn()}, ctx.Expr)) {
+ if (!UpdateLambdaAllArgumentsTypes(lambda, {itemType, input->Child(1)->GetTypeAnn()}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*lambda->GetTypeAnn(), *input->Child(1)->GetTypeAnn())) {
+ if (!IsSameAnnotation(*lambda->GetTypeAnn(), *input->Child(1)->GetTypeAnn())) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, "
- << *lambda->GetTypeAnn() << " != " << *input->Child(1)->GetTypeAnn()));
+ << *lambda->GetTypeAnn() << " != " << *input->Child(1)->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(input->Child(1)->GetTypeAnn());
+ input->SetTypeAnn(input->Child(1)->GetTypeAnn());
return IGraphTransformer::TStatus::Ok;
}
@@ -3322,7 +3322,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!initLambda->GetTypeAnn()) {
+ if (!initLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -3332,18 +3332,18 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!updateLambda->GetTypeAnn()) {
+ if (!updateLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, "
- << *updateLambda->GetTypeAnn() << "!= " << *stateType));
+ << *updateLambda->GetTypeAnn() << "!= " << *stateType));
return IGraphTransformer::TStatus::Error;
}
const auto optAnn = ctx.Expr.MakeType<TOptionalExprType>(stateType);
- input->SetTypeAnn(optAnn);
+ input->SetTypeAnn(optAnn);
return IGraphTransformer::TStatus::Ok;
}
@@ -3361,7 +3361,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- const auto stateType = input->Child(1)->GetTypeAnn();
+ const auto stateType = input->Child(1)->GetTypeAnn();
auto& switchLambda = input->ChildRef(2);
auto& updateLambda = input->ChildRef(3);
@@ -3376,20 +3376,20 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!switchLambda->GetTypeAnn() || !updateLambda->GetTypeAnn()) {
+ if (!switchLambda->GetTypeAnn() || !updateLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
bool isOptional;
const TDataExprType* dataType;
- if (!EnsureDataOrOptionalOfData(switchLambda->Pos(), switchLambda->GetTypeAnn(), isOptional, dataType, ctx.Expr) || !EnsureSpecificDataType(switchLambda->Pos(), *dataType, EDataSlot::Bool, ctx.Expr)) {
+ if (!EnsureDataOrOptionalOfData(switchLambda->Pos(), switchLambda->GetTypeAnn(), isOptional, dataType, ctx.Expr) || !EnsureSpecificDataType(switchLambda->Pos(), *dataType, EDataSlot::Bool, ctx.Expr)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(switchLambda->Pos()), TStringBuilder() << "Switch lambda returns " << *switchLambda->GetTypeAnn() << " instead of boolean."));
return IGraphTransformer::TStatus::Error;
}
- if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, "
- << *updateLambda->GetTypeAnn() << " != " << *stateType));
+ << *updateLambda->GetTypeAnn() << " != " << *stateType));
return IGraphTransformer::TStatus::Error;
}
@@ -3422,31 +3422,31 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!initLambda->GetTypeAnn()) {
+ if (!initLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- const auto stateType = initLambda->GetTypeAnn();
+ const auto stateType = initLambda->GetTypeAnn();
if (!UpdateLambdaAllArgumentsTypes(switchLambda, { itemType, stateType }, ctx.Expr) ||
!UpdateLambdaAllArgumentsTypes(updateLambda, { itemType, stateType }, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!switchLambda->GetTypeAnn() || !updateLambda->GetTypeAnn()) {
+ if (!switchLambda->GetTypeAnn() || !updateLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
bool isOptional;
const TDataExprType* dataType;
- if (!EnsureDataOrOptionalOfData(switchLambda->Pos(), switchLambda->GetTypeAnn(), isOptional, dataType, ctx.Expr) || !EnsureSpecificDataType(switchLambda->Pos(), *dataType, EDataSlot::Bool, ctx.Expr)) {
+ if (!EnsureDataOrOptionalOfData(switchLambda->Pos(), switchLambda->GetTypeAnn(), isOptional, dataType, ctx.Expr) || !EnsureSpecificDataType(switchLambda->Pos(), *dataType, EDataSlot::Bool, ctx.Expr)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(switchLambda->Pos()), TStringBuilder() << "Switch lambda returns " << *switchLambda->GetTypeAnn() << " instead of boolean."));
return IGraphTransformer::TStatus::Error;
}
- if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, "
- << *updateLambda->GetTypeAnn() << "!= " << *stateType));
+ << *updateLambda->GetTypeAnn() << "!= " << *stateType));
return IGraphTransformer::TStatus::Error;
}
@@ -3475,18 +3475,18 @@ namespace {
return status;
}
- auto stateType = input->Child(1)->GetTypeAnn();
+ auto stateType = input->Child(1)->GetTypeAnn();
auto itemType = input->Head().GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
if (!UpdateLambdaAllArgumentsTypes(lambda, { itemType, stateType }, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*lambda->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*lambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, "
- << *lambda->GetTypeAnn() << " != " << *stateType));
+ << *lambda->GetTypeAnn() << " != " << *stateType));
return IGraphTransformer::TStatus::Error;
}
@@ -3509,11 +3509,11 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!saveLambda->GetTypeAnn()) {
+ if (!saveLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto savedStateType = saveLambda->GetTypeAnn();
+ auto savedStateType = saveLambda->GetTypeAnn();
if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -3521,13 +3521,13 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!loadLambda->GetTypeAnn()) {
+ if (!loadLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, "
- << *loadLambda->GetTypeAnn() << " != " << *stateType));
+ << *loadLambda->GetTypeAnn() << " != " << *stateType));
return IGraphTransformer::TStatus::Error;
}
}
@@ -3538,7 +3538,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(stateType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(stateType));
return IGraphTransformer::TStatus::Ok;
}
@@ -3566,22 +3566,22 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!initLambda->GetTypeAnn()) {
+ if (!initLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto stateType = initLambda->GetTypeAnn();
+ auto stateType = initLambda->GetTypeAnn();
if (!UpdateLambdaAllArgumentsTypes(updateLambda, { itemType, stateType }, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!updateLambda->GetTypeAnn()) {
+ if (!updateLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*updateLambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateLambda->Pos()), TStringBuilder() << "Mismatch of lambda return type and state type, "
- << *updateLambda->GetTypeAnn() << "!= " << *stateType));
+ << *updateLambda->GetTypeAnn() << "!= " << *stateType));
return IGraphTransformer::TStatus::Error;
}
@@ -3604,11 +3604,11 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!saveLambda->GetTypeAnn()) {
+ if (!saveLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto savedStateType = saveLambda->GetTypeAnn();
+ auto savedStateType = saveLambda->GetTypeAnn();
if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -3616,13 +3616,13 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!loadLambda->GetTypeAnn()) {
+ if (!loadLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, "
- << *loadLambda->GetTypeAnn() << " != " << *stateType));
+ << *loadLambda->GetTypeAnn() << " != " << *stateType));
return IGraphTransformer::TStatus::Error;
}
}
@@ -3634,7 +3634,7 @@ namespace {
}
auto optAnn = ctx.Expr.MakeType<TStreamExprType>(stateType);
- input->SetTypeAnn(optAnn);
+ input->SetTypeAnn(optAnn);
return IGraphTransformer::TStatus::Ok;
}
@@ -3705,7 +3705,7 @@ namespace {
}
auto retTuple = ctx.Expr.MakeType<TTupleExprType>(tupleItems);
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(retTuple));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(retTuple));
return IGraphTransformer::TStatus::Ok;
}
@@ -3756,7 +3756,7 @@ namespace {
tupleItems[0] = expectedType;
tupleItems[1] = input->Head().GetTypeAnn()->Cast<TListExprType>()->GetItemType();
auto retTuple = ctx.Expr.MakeType<TTupleExprType>(tupleItems);
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(retTuple));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(retTuple));
return IGraphTransformer::TStatus::Ok;
}
@@ -3782,14 +3782,14 @@ namespace {
auto listType = type->Cast<TListExprType>();
auto itemType = listType->GetItemType();
for (size_t i = 1; i < input->ChildrenSize(); ++i) {
- if (!IsSameAnnotation(*itemType, *input->Child(i)->GetTypeAnn())) {
+ if (!IsSameAnnotation(*itemType, *input->Child(i)->GetTypeAnn())) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(i)->Pos()), TStringBuilder() << "Mismatch type of list item, "
- << *itemType << " != " << *input->Child(i)->GetTypeAnn()));
+ << *itemType << " != " << *input->Child(i)->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
}
- input->SetTypeAnn(listType);
+ input->SetTypeAnn(listType);
return IGraphTransformer::TStatus::Ok;
}
@@ -3819,7 +3819,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaPreMap->GetTypeAnn()) {
+ if (!lambdaPreMap->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -3833,7 +3833,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaKeySelector->GetTypeAnn()) {
+ if (!lambdaKeySelector->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -3841,52 +3841,52 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (lambdaKeySelector->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple &&
- lambdaKeySelector->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() < 2) {
+ if (lambdaKeySelector->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple &&
+ lambdaKeySelector->GetTypeAnn()->Cast<TTupleExprType>()->GetSize() < 2) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaKeySelector->Pos()), "Tuple must contain at least 2 items"));
return IGraphTransformer::TStatus::Error;
}
auto& lambdaInitHandler = input->ChildRef(3);
- if (!UpdateLambdaAllArgumentsTypes(lambdaInitHandler, {lambdaKeySelector->GetTypeAnn(), itemType}, ctx.Expr)) {
+ if (!UpdateLambdaAllArgumentsTypes(lambdaInitHandler, {lambdaKeySelector->GetTypeAnn(), itemType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaInitHandler->GetTypeAnn()) {
+ if (!lambdaInitHandler->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!EnsureComputableType(lambdaInitHandler->Pos(), *lambdaInitHandler->GetTypeAnn(), ctx.Expr)) {
+ if (!EnsureComputableType(lambdaInitHandler->Pos(), *lambdaInitHandler->GetTypeAnn(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- auto stateType = lambdaInitHandler->GetTypeAnn();
+ auto stateType = lambdaInitHandler->GetTypeAnn();
auto& lambdaUpdateHandler = input->ChildRef(4);
- if (!UpdateLambdaAllArgumentsTypes(lambdaUpdateHandler, {lambdaKeySelector->GetTypeAnn(), itemType, stateType}, ctx.Expr)) {
+ if (!UpdateLambdaAllArgumentsTypes(lambdaUpdateHandler, {lambdaKeySelector->GetTypeAnn(), itemType, stateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaUpdateHandler->GetTypeAnn()) {
+ if (!lambdaUpdateHandler->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*lambdaUpdateHandler->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*lambdaUpdateHandler->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdateHandler->Pos()), TStringBuilder() <<
- "Mismatch of update lambda return type and state type, " << *lambdaUpdateHandler->GetTypeAnn() << "!= "
+ "Mismatch of update lambda return type and state type, " << *lambdaUpdateHandler->GetTypeAnn() << "!= "
<< *stateType));
return IGraphTransformer::TStatus::Error;
}
auto& lambdaFinishHandler = input->ChildRef(5);
- if (!UpdateLambdaAllArgumentsTypes(lambdaFinishHandler, {lambdaKeySelector->GetTypeAnn(), stateType}, ctx.Expr)) {
+ if (!UpdateLambdaAllArgumentsTypes(lambdaFinishHandler, {lambdaKeySelector->GetTypeAnn(), stateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaFinishHandler->GetTypeAnn()) {
+ if (!lambdaFinishHandler->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!EnsureComputableType(lambdaFinishHandler->Pos(), *lambdaFinishHandler->GetTypeAnn(), ctx.Expr)) {
+ if (!EnsureComputableType(lambdaFinishHandler->Pos(), *lambdaFinishHandler->GetTypeAnn(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -3960,9 +3960,9 @@ namespace {
}
if (firstArgKind == ETypeAnnotationKind::List) {
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(extractedType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(extractedType));
} else {
- input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(extractedType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TOptionalExprType>(extractedType));
}
return IGraphTransformer::TStatus::Ok;
@@ -4008,38 +4008,38 @@ namespace {
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus AssumeUniqueWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ IGraphTransformer::TStatus AssumeUniqueWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
const auto status = NormalizeTupleOfAtoms(input, 1, output, ctx.Expr);
- if (status != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
- const TTypeAnnotationNode* inputItemType = nullptr;
- if (!EnsureNewSeqType<false>(input->Head(), ctx.Expr, &inputItemType)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- if (!EnsureStructType(input->Head().Pos(), *inputItemType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- const auto structType = inputItemType->Cast<TStructExprType>();
- for (auto& x : input->Tail().Children()) {
- YQL_ENSURE(x->IsAtom());
+ if (status != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
+ const TTypeAnnotationNode* inputItemType = nullptr;
+ if (!EnsureNewSeqType<false>(input->Head(), ctx.Expr, &inputItemType)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ if (!EnsureStructType(input->Head().Pos(), *inputItemType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ const auto structType = inputItemType->Cast<TStructExprType>();
+ for (auto& x : input->Tail().Children()) {
+ YQL_ENSURE(x->IsAtom());
auto pos = FindOrReportMissingMember(x->Content(), input->Head().Pos(), *structType, ctx);
- if (!pos) {
- return IGraphTransformer::TStatus::Error;
- }
- }
-
- input->SetTypeAnn(input->Head().GetTypeAnn());
- return IGraphTransformer::TStatus::Ok;
- }
-
+ if (!pos) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ }
+
+ input->SetTypeAnn(input->Head().GetTypeAnn());
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus AssumeColumnOrderWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx) {
if (!EnsureArgsCount(*input, 2, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
@@ -4165,11 +4165,11 @@ namespace {
}
}
- if (!lambdaInit->GetTypeAnn()) {
+ if (!lambdaInit->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto combineStateType = lambdaInit->GetTypeAnn();
+ auto combineStateType = lambdaInit->GetTypeAnn();
if (!EnsureComputableType(lambdaInit->Pos(), *combineStateType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -4186,13 +4186,13 @@ namespace {
}
}
- if (!lambdaUpdate->GetTypeAnn()) {
+ if (!lambdaUpdate->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *combineStateType)) {
+ if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *combineStateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: "
- << *combineStateType << ", but got: " << *lambdaUpdate->GetTypeAnn()));
+ << *combineStateType << ", but got: " << *lambdaUpdate->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -4201,11 +4201,11 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaSave->GetTypeAnn()) {
+ if (!lambdaSave->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto savedType = lambdaSave->GetTypeAnn();
+ auto savedType = lambdaSave->GetTypeAnn();
if (!EnsurePersistableType(lambdaSave->Pos(), *savedType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -4215,7 +4215,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaLoad->GetTypeAnn()) {
+ if (!lambdaLoad->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -4225,19 +4225,19 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- auto reduceStateType = lambdaLoad->GetTypeAnn();
+ auto reduceStateType = lambdaLoad->GetTypeAnn();
auto& lambdaMerge = input->ChildRef(5);
if (!UpdateLambdaAllArgumentsTypes(lambdaMerge, { reduceStateType, reduceStateType }, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaMerge->GetTypeAnn()) {
+ if (!lambdaMerge->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *reduceStateType)) {
+ if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *reduceStateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: "
- << *reduceStateType << ", but got: " << *lambdaMerge->GetTypeAnn()));
+ << *reduceStateType << ", but got: " << *lambdaMerge->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -4246,11 +4246,11 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaFinish->GetTypeAnn()) {
+ if (!lambdaFinish->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto finishType = lambdaFinish->GetTypeAnn();
+ auto finishType = lambdaFinish->GetTypeAnn();
if (!EnsureComputableType(lambdaFinish->Pos(), *finishType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -4260,7 +4260,7 @@ namespace {
return status;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
+ input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
return IGraphTransformer::TStatus::Ok;
}
@@ -4290,7 +4290,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!exprLambda->GetTypeAnn()) {
+ if (!exprLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -4528,7 +4528,7 @@ namespace {
}
}
- auto finishType = child->Child(1)->Child(6)->GetTypeAnn();
+ auto finishType = child->Child(1)->Child(6)->GetTypeAnn();
bool isOptional = finishType->GetKind() == ETypeAnnotationKind::Optional;
if (child->Head().IsList()) {
if (isOptional) {
@@ -4571,9 +4571,9 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(isStream
+ input->SetTypeAnn(isStream
? (const TTypeAnnotationNode*)ctx.Expr.MakeType<TStreamExprType>(rowType)
- : (const TTypeAnnotationNode*)ctx.Expr.MakeType<TListExprType>(rowType));
+ : (const TTypeAnnotationNode*)ctx.Expr.MakeType<TListExprType>(rowType));
return IGraphTransformer::TStatus::Ok;
}
@@ -4958,7 +4958,7 @@ namespace {
}
}
- input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
+ input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
return IGraphTransformer::TStatus::Ok;
}
@@ -5012,11 +5012,11 @@ namespace {
}
}
- if (!lambdaInit->GetTypeAnn()) {
+ if (!lambdaInit->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto stateType = lambdaInit->GetTypeAnn();
+ auto stateType = lambdaInit->GetTypeAnn();
if (!EnsureComputableType(lambdaInit->Pos(), *stateType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -5031,13 +5031,13 @@ namespace {
}
}
- if (!lambdaUpdate->GetTypeAnn()) {
+ if (!lambdaUpdate->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: "
- << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn()));
+ << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -5051,7 +5051,7 @@ namespace {
}
}
- if (!lambdaShift->GetTypeAnn()) {
+ if (!lambdaShift->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -5091,11 +5091,11 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaCurrent->GetTypeAnn()) {
+ if (!lambdaCurrent->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto currentType = lambdaCurrent->GetTypeAnn();
+ auto currentType = lambdaCurrent->GetTypeAnn();
if (!EnsureComputableType(lambdaCurrent->Pos(), *currentType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -5105,7 +5105,7 @@ namespace {
return status;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
+ input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
return IGraphTransformer::TStatus::Ok;
}
@@ -5235,7 +5235,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(rowType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(rowType));
return IGraphTransformer::TStatus::Ok;
}
@@ -5274,11 +5274,11 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(lambdaValue, {rowType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaValue->GetTypeAnn()) {
+ if (!lambdaValue->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
const bool isOptional = lambdaValue->GetTypeAnn()->IsOptionalOrNull();
- input->SetTypeAnn(isOptional ? lambdaValue->GetTypeAnn() : ctx.Expr.MakeType<TOptionalExprType>(lambdaValue->GetTypeAnn()));
+ input->SetTypeAnn(isOptional ? lambdaValue->GetTypeAnn() : ctx.Expr.MakeType<TOptionalExprType>(lambdaValue->GetTypeAnn()));
return IGraphTransformer::TStatus::Ok;
}
@@ -5290,7 +5290,7 @@ namespace {
if (auto status = EnsureTypeRewrite(input->HeadRef(), ctx.Expr); status != IGraphTransformer::TStatus::Ok) {
return status;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Uint64));
return IGraphTransformer::TStatus::Ok;
}
@@ -5420,23 +5420,23 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(lambdaTimeExtractor, {itemType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaTimeExtractor->GetTypeAnn()) {
+ if (!lambdaTimeExtractor->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
const TTypeAnnotationNode* timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp));
- if (!IsSameAnnotation(*lambdaTimeExtractor->GetTypeAnn(), *timeType)) {
+ if (!IsSameAnnotation(*lambdaTimeExtractor->GetTypeAnn(), *timeType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaTimeExtractor->Pos()), TStringBuilder()
<< "Mismatch hopping window time extractor lambda output type, expected: "
- << *timeType << ", but got: " << *lambdaTimeExtractor->GetTypeAnn()));
+ << *timeType << ", but got: " << *lambdaTimeExtractor->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
const TTypeAnnotationNode* intervalType = ctx.Expr.MakeType<TDataExprType>(EDataSlot::Interval);
auto checkWindowParam = [&] (TExprNode::TPtr& param) -> IGraphTransformer::TStatus {
- auto type = param->GetTypeAnn();
+ auto type = param->GetTypeAnn();
if (type->GetKind() == ETypeAnnotationKind::Optional) {
if (param->IsCallable("Nothing")) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(param->Pos()), "Hopping window parameter value cannot be evaluated"));
@@ -5480,7 +5480,7 @@ namespace {
}
}
- input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
+ input->SetTypeAnn(ctx.Expr.MakeType<TUnitExprType>());
return IGraphTransformer::TStatus::Ok;
}
@@ -5518,7 +5518,7 @@ namespace {
}
auto timeType = ctx.Expr.MakeType<TOptionalExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Timestamp));
- auto itemType = item->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
+ auto itemType = item->GetTypeAnn()->Cast<TStreamExprType>()->GetItemType();
if (!EnsureStructType(input->Pos(), *itemType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
@@ -5545,11 +5545,11 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(lambdaInit, {itemType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaInit->GetTypeAnn()) {
+ if (!lambdaInit->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto stateType = lambdaInit->GetTypeAnn();
+ auto stateType = lambdaInit->GetTypeAnn();
if (!EnsureComputableType(lambdaInit->Pos(), *stateType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -5557,38 +5557,38 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(lambdaUpdate, {itemType, stateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaUpdate->GetTypeAnn()) {
+ if (!lambdaUpdate->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*lambdaUpdate->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaUpdate->Pos()), TStringBuilder() << "Mismatch update lambda result type, expected: "
- << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn()));
+ << *stateType << ", but got: " << *lambdaUpdate->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
if (!UpdateLambdaAllArgumentsTypes(lambdaMerge, {stateType, stateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaMerge->GetTypeAnn()) {
+ if (!lambdaMerge->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*lambdaMerge->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaMerge->Pos()), TStringBuilder() << "Mismatch merge lambda result type, expected: "
- << *stateType << ", but got: " << *lambdaMerge->GetTypeAnn()));
+ << *stateType << ", but got: " << *lambdaMerge->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
if (!UpdateLambdaAllArgumentsTypes(lambdaFinish, {stateType, timeType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!lambdaFinish->GetTypeAnn()) {
+ if (!lambdaFinish->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (lambdaFinish->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
+ if (lambdaFinish->GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(lambdaFinish->Pos()), TStringBuilder() <<
- "Expected struct type as finish lambda result type, but got: " << *lambdaMerge->GetTypeAnn()));
+ "Expected struct type as finish lambda result type, but got: " << *lambdaMerge->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -5608,11 +5608,11 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(saveLambda, {stateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!saveLambda->GetTypeAnn()) {
+ if (!saveLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- auto savedStateType = saveLambda->GetTypeAnn();
+ auto savedStateType = saveLambda->GetTypeAnn();
if (!EnsurePersistableType(saveLambda->Pos(), *savedStateType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -5620,18 +5620,18 @@ namespace {
if (!UpdateLambdaAllArgumentsTypes(loadLambda, {savedStateType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!loadLambda->GetTypeAnn()) {
+ if (!loadLambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
- if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
+ if (!IsSameAnnotation(*loadLambda->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(loadLambda->Pos()), TStringBuilder() << "Mismatch of load lambda return type and state type, "
- << *loadLambda->GetTypeAnn() << " != " << *stateType));
+ << *loadLambda->GetTypeAnn() << " != " << *stateType));
return IGraphTransformer::TStatus::Error;
}
}
- input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(lambdaFinish->GetTypeAnn()));
+ input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(lambdaFinish->GetTypeAnn()));
return IGraphTransformer::TStatus::Ok;
}
@@ -5796,187 +5796,187 @@ namespace {
return IGraphTransformer::TStatus::Ok;
}
- IGraphTransformer::TStatus CombineCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
- Y_UNUSED(output);
+ IGraphTransformer::TStatus CombineCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
+ Y_UNUSED(output);
if (!EnsureMinMaxArgsCount(*input, 5U, 6U, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
const TTypeAnnotationNode* itemType = nullptr;
if (!EnsureNewSeqType<false, false>(input->Head(), ctx.Expr, &itemType)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (TCoCombineCore::idx_MemLimit < input->ChildrenSize()) {
if (!EnsureAtom(*input->Child(TCoCombineCore::idx_MemLimit), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
-
+
ui64 memLimit = 0ULL;
if (!TryFromString(input->Child(TCoCombineCore::idx_MemLimit)->Content(), memLimit)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(TCoCombineCore::idx_MemLimit)->Pos()), TStringBuilder() <<
"Bad memLimit value: " << input->Child(TCoCombineCore::idx_MemLimit)->Content()));
return IGraphTransformer::TStatus::Error;
}
- }
-
- TExprNode::TPtr& keyExtractor = input->ChildRef(TCoCombineCore::idx_KeyExtractor);
- TExprNode::TPtr& initHandler = input->ChildRef(TCoCombineCore::idx_InitHandler);
- TExprNode::TPtr& updateHandler = input->ChildRef(TCoCombineCore::idx_UpdateHandler);
- TExprNode::TPtr& finishHandler = input->ChildRef(TCoCombineCore::idx_FinishHandler);
-
- auto status = ConvertToLambda(keyExtractor, ctx.Expr, 1);
- status = status.Combine(ConvertToLambda(initHandler, ctx.Expr, 2));
- status = status.Combine(ConvertToLambda(updateHandler, ctx.Expr, 3));
- status = status.Combine(ConvertToLambda(finishHandler, ctx.Expr, 2));
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
- // keyExtractor
- if (!UpdateLambdaAllArgumentsTypes(keyExtractor, {itemType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!keyExtractor->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
- if (!EnsureOneOrTupleOfDataOrOptionalOfData(*keyExtractor, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- auto keyType = keyExtractor->GetTypeAnn();
- if (!EnsureHashableKey(keyExtractor->Pos(), keyType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!EnsureEquatableKey(keyExtractor->Pos(), keyType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- // initHandler
- if (!UpdateLambdaAllArgumentsTypes(initHandler, {keyType, itemType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!initHandler->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
- auto stateType = initHandler->GetTypeAnn();
- if (!EnsureComputableType(initHandler->Pos(), *stateType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- // updateHandler
- if (!UpdateLambdaAllArgumentsTypes(updateHandler, {keyType, itemType, stateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!updateHandler->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
- if (!IsSameAnnotation(*updateHandler->GetTypeAnn(), *stateType)) {
+ }
+
+ TExprNode::TPtr& keyExtractor = input->ChildRef(TCoCombineCore::idx_KeyExtractor);
+ TExprNode::TPtr& initHandler = input->ChildRef(TCoCombineCore::idx_InitHandler);
+ TExprNode::TPtr& updateHandler = input->ChildRef(TCoCombineCore::idx_UpdateHandler);
+ TExprNode::TPtr& finishHandler = input->ChildRef(TCoCombineCore::idx_FinishHandler);
+
+ auto status = ConvertToLambda(keyExtractor, ctx.Expr, 1);
+ status = status.Combine(ConvertToLambda(initHandler, ctx.Expr, 2));
+ status = status.Combine(ConvertToLambda(updateHandler, ctx.Expr, 3));
+ status = status.Combine(ConvertToLambda(finishHandler, ctx.Expr, 2));
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
+ // keyExtractor
+ if (!UpdateLambdaAllArgumentsTypes(keyExtractor, {itemType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!keyExtractor->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ if (!EnsureOneOrTupleOfDataOrOptionalOfData(*keyExtractor, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ auto keyType = keyExtractor->GetTypeAnn();
+ if (!EnsureHashableKey(keyExtractor->Pos(), keyType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!EnsureEquatableKey(keyExtractor->Pos(), keyType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ // initHandler
+ if (!UpdateLambdaAllArgumentsTypes(initHandler, {keyType, itemType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!initHandler->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ auto stateType = initHandler->GetTypeAnn();
+ if (!EnsureComputableType(initHandler->Pos(), *stateType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ // updateHandler
+ if (!UpdateLambdaAllArgumentsTypes(updateHandler, {keyType, itemType, stateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!updateHandler->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ if (!IsSameAnnotation(*updateHandler->GetTypeAnn(), *stateType)) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(updateHandler->Pos()), "Mismatch of update lambda return type and state type"));
- return IGraphTransformer::TStatus::Error;
- }
-
- // finishHandler
- if (!UpdateLambdaAllArgumentsTypes(finishHandler, {keyType, stateType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!finishHandler->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
- auto finishType = finishHandler->GetTypeAnn();
- if (!EnsureSeqOrOptionalType(finishHandler->Pos(), *finishType, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- auto retKind = finishType->GetKind();
- const TTypeAnnotationNode* retItemType = nullptr;
- if (retKind == ETypeAnnotationKind::List) {
- retItemType = finishType->Cast<TListExprType>()->GetItemType();
- } else if (retKind == ETypeAnnotationKind::Optional) {
- retItemType = finishType->Cast<TOptionalExprType>()->GetItemType();
- } else {
- retItemType = finishType->Cast<TStreamExprType>()->GetItemType();
- }
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ // finishHandler
+ if (!UpdateLambdaAllArgumentsTypes(finishHandler, {keyType, stateType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!finishHandler->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ auto finishType = finishHandler->GetTypeAnn();
+ if (!EnsureSeqOrOptionalType(finishHandler->Pos(), *finishType, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ auto retKind = finishType->GetKind();
+ const TTypeAnnotationNode* retItemType = nullptr;
+ if (retKind == ETypeAnnotationKind::List) {
+ retItemType = finishType->Cast<TListExprType>()->GetItemType();
+ } else if (retKind == ETypeAnnotationKind::Optional) {
+ retItemType = finishType->Cast<TOptionalExprType>()->GetItemType();
+ } else {
+ retItemType = finishType->Cast<TStreamExprType>()->GetItemType();
+ }
+
input->SetTypeAnn(MakeSequenceType(input->Head().GetTypeAnn()->GetKind(), *retItemType, ctx.Expr));
- return IGraphTransformer::TStatus::Ok;
- }
-
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus GroupingCoreWrapper(const TExprNode::TPtr& input, TExprNode::TPtr&, TContext& ctx) {
- if (!EnsureMinMaxArgsCount(*input, 3, 4, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
+ if (!EnsureMinMaxArgsCount(*input, 3, 4, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+
if (!EnsureStreamType(*input->Child(TCoGroupingCore::idx_Input), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
-
- const bool hasHandler = 4 == input->ChildrenSize();
-
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ const bool hasHandler = 4 == input->ChildrenSize();
+
const auto streamType = input->Child(TCoGroupingCore::idx_Input)->GetTypeAnn();
- auto itemType = streamType->Cast<TStreamExprType>()->GetItemType();
-
+ auto itemType = streamType->Cast<TStreamExprType>()->GetItemType();
+
auto& groupSwitch = input->ChildRef(TCoGroupingCore::idx_GroupSwitch);
auto& keyExtractor = input->ChildRef(TCoGroupingCore::idx_KeyExtractor);
auto status = ConvertToLambda(groupSwitch, ctx.Expr, 2);
- status = status.Combine(ConvertToLambda(keyExtractor, ctx.Expr, 1));
- if (hasHandler) {
- auto& handler = input->ChildRef(TCoGroupingCore::idx_ConvertHandler);
- status = status.Combine(ConvertToLambda(handler, ctx.Expr, 1));
- }
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
-
- // keyExtractor
- if (!UpdateLambdaAllArgumentsTypes(keyExtractor, {itemType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- if (!keyExtractor->GetTypeAnn()) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- if (!keyExtractor->GetTypeAnn()->IsHashable() || !keyExtractor->GetTypeAnn()->IsEquatable()) {
+ status = status.Combine(ConvertToLambda(keyExtractor, ctx.Expr, 1));
+ if (hasHandler) {
+ auto& handler = input->ChildRef(TCoGroupingCore::idx_ConvertHandler);
+ status = status.Combine(ConvertToLambda(handler, ctx.Expr, 1));
+ }
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+
+ // keyExtractor
+ if (!UpdateLambdaAllArgumentsTypes(keyExtractor, {itemType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ if (!keyExtractor->GetTypeAnn()) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ if (!keyExtractor->GetTypeAnn()->IsHashable() || !keyExtractor->GetTypeAnn()->IsEquatable()) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(keyExtractor->Pos()), TStringBuilder()
- << "Expected hashable and equatable type for key, but got: " << *keyExtractor->GetTypeAnn()));
- return IGraphTransformer::TStatus::Error;
- }
-
+ << "Expected hashable and equatable type for key, but got: " << *keyExtractor->GetTypeAnn()));
+ return IGraphTransformer::TStatus::Error;
+ }
+
// groupSwitch
- auto keyType = keyExtractor->GetTypeAnn();
+ auto keyType = keyExtractor->GetTypeAnn();
if (!UpdateLambdaAllArgumentsTypes(groupSwitch, {keyType, itemType}, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- if (!groupSwitch->GetTypeAnn()) {
+ if (!groupSwitch->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
if (!EnsureSpecificDataType(*groupSwitch, EDataSlot::Bool, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- // handler
- if (hasHandler) {
- auto& handler = input->ChildRef(TCoGroupingCore::idx_ConvertHandler);
- if (!UpdateLambdaAllArgumentsTypes(handler, {itemType}, ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- itemType = handler->GetTypeAnn();
- if (!itemType) {
- return IGraphTransformer::TStatus::Repeat;
- }
- }
-
- input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(
- ctx.Expr.MakeType<TTupleExprType>(
- TTypeAnnotationNode::TListType{
- keyExtractor->GetTypeAnn(),
- ctx.Expr.MakeType<TStreamExprType>(itemType)
- }
- )
- ));
- return IGraphTransformer::TStatus::Ok;
- }
-
+ // handler
+ if (hasHandler) {
+ auto& handler = input->ChildRef(TCoGroupingCore::idx_ConvertHandler);
+ if (!UpdateLambdaAllArgumentsTypes(handler, {itemType}, ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ itemType = handler->GetTypeAnn();
+ if (!itemType) {
+ return IGraphTransformer::TStatus::Repeat;
+ }
+ }
+
+ input->SetTypeAnn(ctx.Expr.MakeType<TStreamExprType>(
+ ctx.Expr.MakeType<TTupleExprType>(
+ TTypeAnnotationNode::TListType{
+ keyExtractor->GetTypeAnn(),
+ ctx.Expr.MakeType<TStreamExprType>(itemType)
+ }
+ )
+ ));
+ return IGraphTransformer::TStatus::Ok;
+ }
+
IGraphTransformer::TStatus ChopperWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx) {
if (!EnsureArgsCount(*input, 4U, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
diff --git a/ydb/library/yql/core/type_ann/type_ann_list.h b/ydb/library/yql/core/type_ann/type_ann_list.h
index 608b813fe8..7d6f52d094 100644
--- a/ydb/library/yql/core/type_ann/type_ann_list.h
+++ b/ydb/library/yql/core/type_ann/type_ann_list.h
@@ -64,7 +64,7 @@ namespace NTypeAnnImpl {
IGraphTransformer::TStatus SortWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus TopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus KeepTopWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
- IGraphTransformer::TStatus UnorderedWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
+ IGraphTransformer::TStatus UnorderedWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus SortTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus SessionWindowTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus GroupByKeyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
@@ -86,7 +86,7 @@ namespace NTypeAnnImpl {
IGraphTransformer::TStatus CombineByKeyWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus ExtractWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus ExtractMembersWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
- IGraphTransformer::TStatus AssumeUniqueWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
+ IGraphTransformer::TStatus AssumeUniqueWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus AssumeColumnOrderWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExtContext& ctx);
IGraphTransformer::TStatus AggregationTraitsWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
IGraphTransformer::TStatus MultiAggregateWrapper(const TExprNode::TPtr& input, TExprNode::TPtr& output, TContext& ctx);
diff --git a/ydb/library/yql/core/type_ann/type_ann_types.cpp b/ydb/library/yql/core/type_ann/type_ann_types.cpp
index c27c0c433f..65344268bf 100644
--- a/ydb/library/yql/core/type_ann/type_ann_types.cpp
+++ b/ydb/library/yql/core/type_ann/type_ann_types.cpp
@@ -55,20 +55,20 @@ namespace NTypeAnnImpl {
}
IGraphTransformer::TStatus EnsureCodeOrListOfCode(TExprNode::TPtr& node, TExprContext& ctx) {
- if (!node->GetTypeAnn()) {
+ if (!node->GetTypeAnn()) {
ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Expected code or list of code, but got lambda"));
return IGraphTransformer::TStatus::Error;
}
auto resType = MakeCodeResourceType(ctx);
- if (IsSameAnnotation(*node->GetTypeAnn(), *resType)) {
+ if (IsSameAnnotation(*node->GetTypeAnn(), *resType)) {
// wrap with AsList
node = ctx.NewCallable(node->Pos(), "AsList", { node });
return IGraphTransformer::TStatus::Repeat;
}
auto lstType = ctx.MakeType<TListExprType>(resType);
- if (IsSameAnnotation(*node->GetTypeAnn(), *lstType)) {
+ if (IsSameAnnotation(*node->GetTypeAnn(), *lstType)) {
return IGraphTransformer::TStatus::Ok;
}
@@ -89,7 +89,7 @@ namespace NTypeAnnImpl {
}
auto genType = ctx.Expr.MakeType<TGenericExprType>();
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(genType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(genType));
return IGraphTransformer::TStatus::Ok;
}
@@ -106,7 +106,7 @@ namespace NTypeAnnImpl {
auto tag = input->Child(0)->Content();
auto resType = ctx.Expr.MakeType<TResourceExprType>(tag);
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(resType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(resType));
return IGraphTransformer::TStatus::Ok;
}
@@ -125,7 +125,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- auto underlyingType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto underlyingType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureInspectableType(input->Child(0)->Pos(), *underlyingType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -133,10 +133,10 @@ namespace NTypeAnnImpl {
auto tag = input->Child(1)->Content();
auto taggedType = ctx.Expr.MakeType<TTaggedExprType>(underlyingType, tag);
- if (!taggedType->Validate(input->Pos(), ctx.Expr)) {
- return IGraphTransformer::TStatus::Error;
- }
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(taggedType));
+ if (!taggedType->Validate(input->Pos(), ctx.Expr)) {
+ return IGraphTransformer::TStatus::Error;
+ }
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(taggedType));
return IGraphTransformer::TStatus::Ok;
}
@@ -178,7 +178,7 @@ namespace NTypeAnnImpl {
auto file = TString(input->Child(2)->Content());
auto message = input->Child(3)->Content();
auto resType = ctx.Expr.MakeType<TErrorExprType>(TIssue(TPosition(column, row, file), message));
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(resType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(resType));
return IGraphTransformer::TStatus::Ok;
}
@@ -207,14 +207,14 @@ namespace NTypeAnnImpl {
if (!dataType->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(dataType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(dataType));
} else {
if (!EnsureArgsCount(*input, 1, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
auto dataType = ctx.Expr.MakeType<TDataExprType>(*slot);
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(dataType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(dataType));
}
return IGraphTransformer::TStatus::Ok;
}
@@ -230,13 +230,13 @@ namespace NTypeAnnImpl {
return status;
}
- auto itemType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto itemType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureInspectableType(input->Child(0)->Pos(), *itemType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
auto listType = ctx.Expr.MakeType<TListExprType>(itemType);
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(listType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(listType));
return IGraphTransformer::TStatus::Ok;
}
@@ -251,13 +251,13 @@ namespace NTypeAnnImpl {
return status;
}
- auto itemType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto itemType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureInspectableType(input->Child(0)->Pos(), *itemType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
auto listType = ctx.Expr.MakeType<TStreamExprType>(itemType);
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(listType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(listType));
return IGraphTransformer::TStatus::Ok;
}
@@ -293,13 +293,13 @@ namespace NTypeAnnImpl {
return status;
}
- auto itemType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto itemType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureInspectableType(input->Child(0)->Pos(), *itemType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
auto optionalType = ctx.Expr.MakeType<TOptionalExprType>(itemType);
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(optionalType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(optionalType));
return IGraphTransformer::TStatus::Ok;
}
@@ -312,7 +312,7 @@ namespace NTypeAnnImpl {
if (auto status = EnsureTypeRewrite(child, ctx.Expr); status != IGraphTransformer::TStatus::Ok) {
return status;
}
- auto elemType = child->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto elemType = child->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureInspectableType(child->Pos(), *elemType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -321,7 +321,7 @@ namespace NTypeAnnImpl {
}
auto tupleType = ctx.Expr.MakeType<TTupleExprType>(items);
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(tupleType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(tupleType));
return IGraphTransformer::TStatus::Ok;
}
@@ -371,7 +371,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto memberType = typeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto memberType = typeNode->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureInspectableType(typeNode->Pos(), *memberType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -395,7 +395,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(structType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(structType));
return IGraphTransformer::TStatus::Ok;
}
@@ -414,8 +414,8 @@ namespace NTypeAnnImpl {
return status;
}
- auto keyType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
- auto payloadType = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto keyType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto payloadType = input->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureInspectableType(input->Child(1)->Pos(), *payloadType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -425,7 +425,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(dictType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(dictType));
return IGraphTransformer::TStatus::Ok;
}
@@ -437,7 +437,7 @@ namespace NTypeAnnImpl {
}
auto voidType = ctx.Expr.MakeType<TVoidExprType>();
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(voidType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(voidType));
return IGraphTransformer::TStatus::Ok;
}
@@ -449,7 +449,7 @@ namespace NTypeAnnImpl {
}
auto voidType = ctx.Expr.MakeType<TNullExprType>();
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(voidType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(voidType));
return IGraphTransformer::TStatus::Ok;
}
@@ -461,7 +461,7 @@ namespace NTypeAnnImpl {
}
auto unitType = ctx.Expr.MakeType<TUnitExprType>();
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(unitType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(unitType));
return IGraphTransformer::TStatus::Ok;
}
@@ -496,8 +496,8 @@ namespace NTypeAnnImpl {
}
auto resType = MakeTypeHandleResourceType(ctx.Expr);
- if (input->Child(0)->GetTypeAnn() == resType) {
- input->SetTypeAnn(resType);
+ if (input->Child(0)->GetTypeAnn() == resType) {
+ input->SetTypeAnn(resType);
return IGraphTransformer::TStatus::Ok;
}
@@ -505,7 +505,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() == ETypeAnnotationKind::Null) {
output = ExpandType(input->Pos(), *ctx.Expr.MakeType<TUnitExprType>(), ctx.Expr);
return IGraphTransformer::TStatus::Repeat;
@@ -528,8 +528,8 @@ namespace NTypeAnnImpl {
}
auto resType = MakeTypeHandleResourceType(ctx.Expr);
- if (input->Child(0)->GetTypeAnn() == resType) {
- input->SetTypeAnn(resType);
+ if (input->Child(0)->GetTypeAnn() == resType) {
+ input->SetTypeAnn(resType);
return IGraphTransformer::TStatus::Ok;
}
@@ -537,7 +537,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() == ETypeAnnotationKind::EmptyList) {
output = ExpandType(input->Pos(), *ctx.Expr.MakeType<TUnitExprType>(), ctx.Expr);
return IGraphTransformer::TStatus::Repeat;
@@ -560,8 +560,8 @@ namespace NTypeAnnImpl {
}
auto resType = MakeTypeHandleResourceType(ctx.Expr);
- if (input->Child(0)->GetTypeAnn() == resType) {
- input->SetTypeAnn(resType);
+ if (input->Child(0)->GetTypeAnn() == resType) {
+ input->SetTypeAnn(resType);
return IGraphTransformer::TStatus::Ok;
}
@@ -569,7 +569,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() != ETypeAnnotationKind::Stream) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()),
TStringBuilder() << "Expected stream type, but got: " << *type));
@@ -594,7 +594,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() != ETypeAnnotationKind::Tuple) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()),
TStringBuilder() << "Expected tuple type, but got: " << *type));
@@ -633,7 +633,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() != ETypeAnnotationKind::Struct) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected struct type, but got: "
<< *type));
@@ -669,7 +669,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() == ETypeAnnotationKind::EmptyDict) {
output = ExpandType(input->Pos(), *ctx.Expr.MakeType<TUnitExprType>(), ctx.Expr);
return IGraphTransformer::TStatus::Repeat;
@@ -695,7 +695,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() == ETypeAnnotationKind::EmptyDict) {
output = ExpandType(input->Pos(), *ctx.Expr.MakeType<TUnitExprType>(), ctx.Expr);
return IGraphTransformer::TStatus::Repeat;
@@ -768,7 +768,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = typeChild->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = typeChild->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureComputableType(child->Pos(), *type, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -808,7 +808,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(callableType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(callableType));
return IGraphTransformer::TStatus::Ok;
}
@@ -822,7 +822,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() != ETypeAnnotationKind::Callable) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected callable type, but got: "
<< *type));
@@ -847,7 +847,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() != ETypeAnnotationKind::Callable) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected callable type, but got: "
<< *type));
@@ -894,7 +894,7 @@ namespace NTypeAnnImpl {
TAstNode::NewList(inputPos, pool,
TAstNode::NewLiteralAtom(inputPos, TStringBuf("return"), pool), parsedType));
TExprNode::TPtr exprRoot;
- if (!CompileExpr(*astRoot, exprRoot, ctx.Expr, nullptr)) {
+ if (!CompileExpr(*astRoot, exprRoot, ctx.Expr, nullptr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -905,7 +905,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(exprRoot->GetTypeAnn());
+ input->SetTypeAnn(exprRoot->GetTypeAnn());
return IGraphTransformer::TStatus::Ok;
}
@@ -922,7 +922,7 @@ namespace NTypeAnnImpl {
}
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
@@ -936,7 +936,7 @@ namespace NTypeAnnImpl {
return status;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -950,7 +950,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Yson));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::Yson));
return IGraphTransformer::TStatus::Ok;
};
@@ -964,7 +964,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -986,7 +986,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureStructType(input->Child(0)->Pos(), *type, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -994,7 +994,7 @@ namespace NTypeAnnImpl {
auto structType = type->Cast<TStructExprType>();
TVector<const TItemExprType*> items = structType->GetItems();
auto newItem = ctx.Expr.MakeType<TItemExprType>(input->Child(1)->Content(),
- input->Child(2)->GetTypeAnn()->Cast<TTypeExprType>()->GetType());
+ input->Child(2)->GetTypeAnn()->Cast<TTypeExprType>()->GetType());
if (!newItem->Validate(input->Pos(), ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -1027,7 +1027,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureStructType(input->Child(0)->Pos(), *type, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -1087,7 +1087,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = child->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = child->Child(1)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureStructOrOptionalStructType(child->Child(1)->Pos(), *type, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -1134,7 +1134,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto underlyingType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto underlyingType = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (!EnsureComputableType(input->Child(0)->Pos(), *underlyingType, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
@@ -1144,7 +1144,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(varType));
+ input->SetTypeAnn(ctx.Expr.MakeType<TTypeExprType>(varType));
return IGraphTransformer::TStatus::Ok;
}
@@ -1155,8 +1155,8 @@ namespace NTypeAnnImpl {
}
auto resType = MakeTypeHandleResourceType(ctx.Expr);
- if (input->Child(0)->GetTypeAnn() == resType) {
- input->SetTypeAnn(resType);
+ if (input->Child(0)->GetTypeAnn() == resType) {
+ input->SetTypeAnn(resType);
return IGraphTransformer::TStatus::Ok;
}
@@ -1164,7 +1164,7 @@ namespace NTypeAnnImpl {
return status;
}
- auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
+ auto type = input->Child(0)->GetTypeAnn()->Cast<TTypeExprType>()->GetType();
if (type->GetKind() != ETypeAnnotationKind::Variant) {
ctx.Expr.AddError(TIssue(ctx.Expr.GetPosition(input->Child(0)->Pos()), TStringBuilder() << "Expected variant type, but got: "
<< *type));
@@ -1185,7 +1185,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
@@ -1200,7 +1200,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String)));
return IGraphTransformer::TStatus::Ok;
}
@@ -1215,12 +1215,12 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- auto itemType = input->Child(0)->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
+ auto itemType = input->Child(0)->GetTypeAnn()->Cast<TListExprType>()->GetItemType();
if (!EnsureSpecificDataType(input->Child(0)->Pos(), *itemType, EDataSlot::String, ctx.Expr)) {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1235,7 +1235,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1250,7 +1250,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1265,7 +1265,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1280,7 +1280,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(MakeTypeHandleResourceType(ctx.Expr)));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(MakeTypeHandleResourceType(ctx.Expr)));
return IGraphTransformer::TStatus::Ok;
}
@@ -1298,7 +1298,7 @@ namespace NTypeAnnImpl {
return status;
}
- input->SetTypeAnn(resType);
+ input->SetTypeAnn(resType);
return IGraphTransformer::TStatus::Ok;
}
@@ -1313,7 +1313,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(MakeItemDescriptorType(ctx.Expr)));
+ input->SetTypeAnn(ctx.Expr.MakeType<TListExprType>(MakeItemDescriptorType(ctx.Expr)));
return IGraphTransformer::TStatus::Ok;
}
@@ -1331,7 +1331,7 @@ namespace NTypeAnnImpl {
return status;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1346,7 +1346,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeDictDescriptorType(ctx.Expr));
+ input->SetTypeAnn(MakeDictDescriptorType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1365,7 +1365,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1380,7 +1380,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
@@ -1395,7 +1395,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1410,7 +1410,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTaggedDescriptorType(ctx.Expr));
+ input->SetTypeAnn(MakeTaggedDescriptorType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1429,7 +1429,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1444,7 +1444,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1455,7 +1455,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1466,7 +1466,7 @@ namespace NTypeAnnImpl {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1566,7 +1566,7 @@ return IGraphTransformer::TStatus::Repeat;
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeCallabledDescriptorType(ctx.Expr));
+ input->SetTypeAnn(MakeCallabledDescriptorType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1607,7 +1607,7 @@ return IGraphTransformer::TStatus::Repeat;
}
}
- input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeTypeHandleResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1621,7 +1621,7 @@ return IGraphTransformer::TStatus::Repeat;
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
+ input->SetTypeAnn(ctx.Expr.MakeType<TDataExprType>(EDataSlot::String));
return IGraphTransformer::TStatus::Ok;
}
@@ -1635,7 +1635,7 @@ return IGraphTransformer::TStatus::Repeat;
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(MakeCodeResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeCodeResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1684,7 +1684,7 @@ return IGraphTransformer::TStatus::Repeat;
return status;
}
- input->SetTypeAnn(MakeCodeResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeCodeResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1701,7 +1701,7 @@ return IGraphTransformer::TStatus::Repeat;
return status;
}
- input->SetTypeAnn(MakeCodeResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeCodeResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1727,7 +1727,7 @@ return IGraphTransformer::TStatus::Repeat;
return status;
}
- input->SetTypeAnn(MakeCodeResourceType(ctx.Expr));
+ input->SetTypeAnn(MakeCodeResourceType(ctx.Expr));
return IGraphTransformer::TStatus::Ok;
}
@@ -1768,7 +1768,7 @@ return IGraphTransformer::TStatus::Repeat;
return IGraphTransformer::TStatus::Error;
}
- if (!lambda->GetTypeAnn()) {
+ if (!lambda->GetTypeAnn()) {
return IGraphTransformer::TStatus::Repeat;
}
@@ -1776,7 +1776,7 @@ return IGraphTransformer::TStatus::Repeat;
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(resType);
+ input->SetTypeAnn(resType);
return IGraphTransformer::TStatus::Ok;
}
} // namespace NTypeAnnImpl
diff --git a/ydb/library/yql/core/type_ann/ya.make b/ydb/library/yql/core/type_ann/ya.make
index be843b5abc..c23858cfbc 100644
--- a/ydb/library/yql/core/type_ann/ya.make
+++ b/ydb/library/yql/core/type_ann/ya.make
@@ -1,10 +1,10 @@
LIBRARY()
-OWNER(
+OWNER(
vvvv
- g:yql
+ g:yql
g:yql_ydb_core
-)
+)
SRCS(
type_ann_core.cpp
diff --git a/ydb/library/yql/core/ya.make b/ydb/library/yql/core/ya.make
index 29169996f5..89eb6cf475 100644
--- a/ydb/library/yql/core/ya.make
+++ b/ydb/library/yql/core/ya.make
@@ -1,22 +1,22 @@
LIBRARY()
-OWNER(
+OWNER(
vvvv
- g:yql
+ g:yql
g:yql_ydb_core
-)
+)
SRCS(
yql_atom_enums.h
- yql_callable_transform.cpp
- yql_callable_transform.h
+ yql_callable_transform.cpp
+ yql_callable_transform.h
yql_csv.cpp
yql_csv.h
yql_data_provider.h
yql_execution.cpp
yql_execution.h
- yql_expr_constraint.cpp
- yql_expr_constraint.h
+ yql_expr_constraint.cpp
+ yql_expr_constraint.h
yql_expr_csee.cpp
yql_expr_csee.h
yql_expr_optimize.cpp
diff --git a/ydb/library/yql/core/yql_callable_names.h b/ydb/library/yql/core/yql_callable_names.h
index c54cc833c4..ccae9d7548 100644
--- a/ydb/library/yql/core/yql_callable_names.h
+++ b/ydb/library/yql/core/yql_callable_names.h
@@ -1,9 +1,9 @@
-#pragma once
-
-#include <util/generic/strbuf.h>
-
-namespace NYql {
-
+#pragma once
+
+#include <util/generic/strbuf.h>
+
+namespace NYql {
+
constexpr TStringBuf LeftName = "Left!";
constexpr TStringBuf RightName = "Right!";
constexpr TStringBuf SyncName = "Sync!";
@@ -14,5 +14,5 @@ constexpr TStringBuf ReadName = "Read!";
constexpr TStringBuf WriteName = "Write!";
constexpr TStringBuf ConfigureName = "Configure!";
constexpr TStringBuf ConsName = "Cons!";
-
-} // namespace NYql
+
+} // namespace NYql
diff --git a/ydb/library/yql/core/yql_callable_transform.cpp b/ydb/library/yql/core/yql_callable_transform.cpp
index 511b86b121..3148b8ea05 100644
--- a/ydb/library/yql/core/yql_callable_transform.cpp
+++ b/ydb/library/yql/core/yql_callable_transform.cpp
@@ -1 +1 @@
-#include "yql_callable_transform.h"
+#include "yql_callable_transform.h"
diff --git a/ydb/library/yql/core/yql_callable_transform.h b/ydb/library/yql/core/yql_callable_transform.h
index d508152029..532bb27cf4 100644
--- a/ydb/library/yql/core/yql_callable_transform.h
+++ b/ydb/library/yql/core/yql_callable_transform.h
@@ -1,284 +1,284 @@
-#pragma once
-
-#include "yql_graph_transformer.h"
-#include "yql_type_annotation.h"
-#include "yql_expr_type_annotation.h"
-#include "yql_callable_names.h"
-
+#pragma once
+
+#include "yql_graph_transformer.h"
+#include "yql_type_annotation.h"
+#include "yql_expr_type_annotation.h"
+#include "yql_callable_names.h"
+
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/public/issue/yql_issue_manager.h>
#include <ydb/library/yql/public/issue/yql_issue.h>
#include <ydb/library/yql/utils/yql_panic.h>
-
+
#include <library/cpp/threading/future/future.h>
-
-#include <util/generic/ptr.h>
-#include <util/string/builder.h>
-
-#include <utility>
-
-namespace NYql {
-
-template <class TDerived>
-class TCallableTransformerBase : public TGraphTransformerBase {
-public:
- TCallableTransformerBase(TTypeAnnotationContext& types, bool instantOnly)
- : Types(types)
- , InstantOnly(instantOnly)
- {}
-
- IGraphTransformer::TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- output = input;
-
+
+#include <util/generic/ptr.h>
+#include <util/string/builder.h>
+
+#include <utility>
+
+namespace NYql {
+
+template <class TDerived>
+class TCallableTransformerBase : public TGraphTransformerBase {
+public:
+ TCallableTransformerBase(TTypeAnnotationContext& types, bool instantOnly)
+ : Types(types)
+ , InstantOnly(instantOnly)
+ {}
+
+ IGraphTransformer::TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ output = input;
+
if (input->IsList()) {
if (const auto maybeStatus = static_cast<TDerived*>(this)->ProcessList(input, output, ctx)) {
return *maybeStatus;
}
}
- auto name = input->Content();
- TIssueScopeGuard issueScope(ctx.IssueManager, [&]() {
+ auto name = input->Content();
+ TIssueScopeGuard issueScope(ctx.IssueManager, [&]() {
return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), TStringBuilder() << "At function: " << name);
- });
-
- TStatus status = TStatus::Ok;
- if (auto maybeStatus = static_cast<TDerived*>(this)->ProcessCore(input, output, ctx)) {
- status = *maybeStatus;
- } else {
- if (name == CommitName) {
- auto datasink = ParseCommit(*input, ctx);
- if (!datasink) {
- status = TStatus::Error;
- } else {
- status = ProcessDataProviderAnnotation(*datasink, input, output, ctx);
- if (status == TStatus::Ok) {
- status = static_cast<TDerived*>(this)->ValidateProviderCommitResult(input, ctx);
- }
- }
- } else if (name == ReadName) {
- auto datasource = ParseRead(*input, ctx);
- if (!datasource) {
- status = TStatus::Error;
- } else {
- status = ProcessDataProviderAnnotation(*datasource, input, output, ctx);
- if (status == TStatus::Ok) {
- status = static_cast<TDerived*>(this)->ValidateProviderReadResult(input, ctx);
- }
- }
- } else if (name == WriteName) {
- auto datasink = ParseWrite(*input, ctx);
- if (!datasink) {
- status = TStatus::Error;
- } else {
- status = ProcessDataProviderAnnotation(*datasink, input, output, ctx);
- if (status == TStatus::Ok) {
- status = static_cast<TDerived*>(this)->ValidateProviderWriteResult(input, ctx);
- }
- }
- } else if (name == ConfigureName) {
- auto provider = ParseConfigure(*input, ctx);
- if (!provider) {
- status = TStatus::Error;
- } else {
- status = ProcessDataProviderAnnotation(*provider, input, output, ctx);
- if (status == TStatus::Ok) {
- status = static_cast<TDerived*>(this)->ValidateProviderConfigureResult(input, ctx);
- }
- }
- } else {
- bool foundFunc = false;
- for (auto& datasource : Types.DataSources) {
- if (!datasource->CanParse(*input)) {
- continue;
- }
-
- foundFunc = true;
- status = ProcessDataProviderAnnotation(*datasource, input, output, ctx);
- break;
- }
-
- if (!foundFunc) {
- for (auto& datasink : Types.DataSinks) {
- if (!datasink->CanParse(*input)) {
- continue;
- }
-
- foundFunc = true;
- status = ProcessDataProviderAnnotation(*datasink, input, output, ctx);
- break;
- }
- }
-
- if (!foundFunc) {
- return static_cast<TDerived*>(this)->ProcessUnknown(input, ctx);
- }
- }
- }
-
- return status;
- }
-
- NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final {
- const auto it = PendingNodes.find(&input);
- YQL_ENSURE(it != PendingNodes.cend());
- return static_cast<TDerived*>(this)->GetTransformer(*it->second.second).GetAsyncFuture(input);
- }
-
- TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- const auto it = PendingNodes.find(input.Get());
- YQL_ENSURE(it != PendingNodes.cend());
- const auto provider = it->second.second;
- IGraphTransformer& transformer = static_cast<TDerived*>(this)->GetTransformer(*provider);
- const auto status = transformer.ApplyAsyncChanges(it->second.first, output, ctx);
- PendingNodes.erase(it);
- return status;
- }
-
-protected:
- IDataProvider* ParseCommit(const TExprNode& input, TExprContext& ctx) {
- if (!EnsureMinArgsCount(input, 2, ctx)) {
- return nullptr;
- }
-
- if (!EnsureMaxArgsCount(input, 3, ctx)) {
- return nullptr;
- }
-
- if (!EnsureWorldType(*input.Child(0), ctx)) {
- return nullptr;
- }
-
- if (!EnsureDataSink(*input.Child(1), ctx)) {
- return nullptr;
- }
-
- if (input.ChildrenSize() == 3) {
- for (auto& setting : input.Child(2)->Children()) {
- if (!EnsureTupleSize(*setting, 2, ctx)) {
- return nullptr;
- }
-
- auto nameNode = setting->Child(0);
- if (!EnsureAtom(*nameNode, ctx)) {
- return nullptr;
- }
- }
- }
-
- auto datasinkName = input.Child(1)->Child(0)->Content();
- auto datasink = Types.DataSinkMap.FindPtr(datasinkName);
- if (!datasink) {
+ });
+
+ TStatus status = TStatus::Ok;
+ if (auto maybeStatus = static_cast<TDerived*>(this)->ProcessCore(input, output, ctx)) {
+ status = *maybeStatus;
+ } else {
+ if (name == CommitName) {
+ auto datasink = ParseCommit(*input, ctx);
+ if (!datasink) {
+ status = TStatus::Error;
+ } else {
+ status = ProcessDataProviderAnnotation(*datasink, input, output, ctx);
+ if (status == TStatus::Ok) {
+ status = static_cast<TDerived*>(this)->ValidateProviderCommitResult(input, ctx);
+ }
+ }
+ } else if (name == ReadName) {
+ auto datasource = ParseRead(*input, ctx);
+ if (!datasource) {
+ status = TStatus::Error;
+ } else {
+ status = ProcessDataProviderAnnotation(*datasource, input, output, ctx);
+ if (status == TStatus::Ok) {
+ status = static_cast<TDerived*>(this)->ValidateProviderReadResult(input, ctx);
+ }
+ }
+ } else if (name == WriteName) {
+ auto datasink = ParseWrite(*input, ctx);
+ if (!datasink) {
+ status = TStatus::Error;
+ } else {
+ status = ProcessDataProviderAnnotation(*datasink, input, output, ctx);
+ if (status == TStatus::Ok) {
+ status = static_cast<TDerived*>(this)->ValidateProviderWriteResult(input, ctx);
+ }
+ }
+ } else if (name == ConfigureName) {
+ auto provider = ParseConfigure(*input, ctx);
+ if (!provider) {
+ status = TStatus::Error;
+ } else {
+ status = ProcessDataProviderAnnotation(*provider, input, output, ctx);
+ if (status == TStatus::Ok) {
+ status = static_cast<TDerived*>(this)->ValidateProviderConfigureResult(input, ctx);
+ }
+ }
+ } else {
+ bool foundFunc = false;
+ for (auto& datasource : Types.DataSources) {
+ if (!datasource->CanParse(*input)) {
+ continue;
+ }
+
+ foundFunc = true;
+ status = ProcessDataProviderAnnotation(*datasource, input, output, ctx);
+ break;
+ }
+
+ if (!foundFunc) {
+ for (auto& datasink : Types.DataSinks) {
+ if (!datasink->CanParse(*input)) {
+ continue;
+ }
+
+ foundFunc = true;
+ status = ProcessDataProviderAnnotation(*datasink, input, output, ctx);
+ break;
+ }
+ }
+
+ if (!foundFunc) {
+ return static_cast<TDerived*>(this)->ProcessUnknown(input, ctx);
+ }
+ }
+ }
+
+ return status;
+ }
+
+ NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final {
+ const auto it = PendingNodes.find(&input);
+ YQL_ENSURE(it != PendingNodes.cend());
+ return static_cast<TDerived*>(this)->GetTransformer(*it->second.second).GetAsyncFuture(input);
+ }
+
+ TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ const auto it = PendingNodes.find(input.Get());
+ YQL_ENSURE(it != PendingNodes.cend());
+ const auto provider = it->second.second;
+ IGraphTransformer& transformer = static_cast<TDerived*>(this)->GetTransformer(*provider);
+ const auto status = transformer.ApplyAsyncChanges(it->second.first, output, ctx);
+ PendingNodes.erase(it);
+ return status;
+ }
+
+protected:
+ IDataProvider* ParseCommit(const TExprNode& input, TExprContext& ctx) {
+ if (!EnsureMinArgsCount(input, 2, ctx)) {
+ return nullptr;
+ }
+
+ if (!EnsureMaxArgsCount(input, 3, ctx)) {
+ return nullptr;
+ }
+
+ if (!EnsureWorldType(*input.Child(0), ctx)) {
+ return nullptr;
+ }
+
+ if (!EnsureDataSink(*input.Child(1), ctx)) {
+ return nullptr;
+ }
+
+ if (input.ChildrenSize() == 3) {
+ for (auto& setting : input.Child(2)->Children()) {
+ if (!EnsureTupleSize(*setting, 2, ctx)) {
+ return nullptr;
+ }
+
+ auto nameNode = setting->Child(0);
+ if (!EnsureAtom(*nameNode, ctx)) {
+ return nullptr;
+ }
+ }
+ }
+
+ auto datasinkName = input.Child(1)->Child(0)->Content();
+ auto datasink = Types.DataSinkMap.FindPtr(datasinkName);
+ if (!datasink) {
ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName));
- return nullptr;
- }
-
- return (*datasink).Get();
- }
-
- IDataProvider* ParseRead(const TExprNode& input, TExprContext& ctx) {
- if (!EnsureMinArgsCount(input, 2, ctx)) {
- return nullptr;
- }
-
- if (!EnsureWorldType(*input.Child(0), ctx)) {
- return nullptr;
- }
-
- if (!EnsureDataSource(*input.Child(1), ctx)) {
- return nullptr;
- }
-
- auto datasourceName = input.Child(1)->Child(0)->Content();
- auto datasource = Types.DataSourceMap.FindPtr(datasourceName);
- if (!datasource) {
+ return nullptr;
+ }
+
+ return (*datasink).Get();
+ }
+
+ IDataProvider* ParseRead(const TExprNode& input, TExprContext& ctx) {
+ if (!EnsureMinArgsCount(input, 2, ctx)) {
+ return nullptr;
+ }
+
+ if (!EnsureWorldType(*input.Child(0), ctx)) {
+ return nullptr;
+ }
+
+ if (!EnsureDataSource(*input.Child(1), ctx)) {
+ return nullptr;
+ }
+
+ auto datasourceName = input.Child(1)->Child(0)->Content();
+ auto datasource = Types.DataSourceMap.FindPtr(datasourceName);
+ if (!datasource) {
ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasource: " << datasourceName));
- return nullptr;
- }
-
- return (*datasource).Get();
- }
-
- IDataProvider* ParseWrite(const TExprNode& input, TExprContext& ctx) {
- if (!EnsureMinArgsCount(input, 2, ctx)) {
- return nullptr;
- }
-
- if (!EnsureWorldType(*input.Child(0), ctx)) {
- return nullptr;
- }
-
- if (!EnsureDataSink(*input.Child(1), ctx)) {
- return nullptr;
- }
-
- auto datasinkName = input.Child(1)->Child(0)->Content();
- auto datasink = Types.DataSinkMap.FindPtr(datasinkName);
- if (!datasink) {
+ return nullptr;
+ }
+
+ return (*datasource).Get();
+ }
+
+ IDataProvider* ParseWrite(const TExprNode& input, TExprContext& ctx) {
+ if (!EnsureMinArgsCount(input, 2, ctx)) {
+ return nullptr;
+ }
+
+ if (!EnsureWorldType(*input.Child(0), ctx)) {
+ return nullptr;
+ }
+
+ if (!EnsureDataSink(*input.Child(1), ctx)) {
+ return nullptr;
+ }
+
+ auto datasinkName = input.Child(1)->Child(0)->Content();
+ auto datasink = Types.DataSinkMap.FindPtr(datasinkName);
+ if (!datasink) {
ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName));
- return nullptr;
- }
-
- return (*datasink).Get();
- }
-
- IDataProvider* ParseConfigure(const TExprNode& input, TExprContext& ctx) {
- if (!EnsureMinArgsCount(input, 2, ctx)) {
- return nullptr;
- }
-
- if (!EnsureWorldType(*input.Child(0), ctx)) {
- return nullptr;
- }
-
- if (!EnsureDataProvider(*input.Child(1), ctx)) {
- return nullptr;
- }
-
- if (input.Child(1)->IsCallable("DataSource")) {
- auto datasourceName = input.Child(1)->Child(0)->Content();
- auto datasource = Types.DataSourceMap.FindPtr(datasourceName);
- if (!datasource) {
+ return nullptr;
+ }
+
+ return (*datasink).Get();
+ }
+
+ IDataProvider* ParseConfigure(const TExprNode& input, TExprContext& ctx) {
+ if (!EnsureMinArgsCount(input, 2, ctx)) {
+ return nullptr;
+ }
+
+ if (!EnsureWorldType(*input.Child(0), ctx)) {
+ return nullptr;
+ }
+
+ if (!EnsureDataProvider(*input.Child(1), ctx)) {
+ return nullptr;
+ }
+
+ if (input.Child(1)->IsCallable("DataSource")) {
+ auto datasourceName = input.Child(1)->Child(0)->Content();
+ auto datasource = Types.DataSourceMap.FindPtr(datasourceName);
+ if (!datasource) {
ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasource: " << datasourceName));
- return nullptr;
- }
-
- return (*datasource).Get();
- }
-
- if (input.Child(1)->IsCallable("DataSink")) {
- auto datasinkName = input.Child(1)->Child(0)->Content();
- auto datasink = Types.DataSinkMap.FindPtr(datasinkName);
- if (!datasink) {
+ return nullptr;
+ }
+
+ return (*datasource).Get();
+ }
+
+ if (input.Child(1)->IsCallable("DataSink")) {
+ auto datasinkName = input.Child(1)->Child(0)->Content();
+ auto datasink = Types.DataSinkMap.FindPtr(datasinkName);
+ if (!datasink) {
ctx.AddError(TIssue(ctx.GetPosition(input.Pos()), TStringBuilder() << "Unsupported datasink: " << datasinkName));
- return nullptr;
- }
-
- return (*datasink).Get();
- }
-
- YQL_ENSURE(false, "Unexpected provider class");
- }
-
- IGraphTransformer::TStatus ProcessDataProviderAnnotation(IDataProvider& dataProvider,
- const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- auto status = static_cast<TDerived*>(this)->GetTransformer(dataProvider).Transform(input, output, ctx);
- if (status.Level == IGraphTransformer::TStatus::Async) {
- if (InstantOnly) {
+ return nullptr;
+ }
+
+ return (*datasink).Get();
+ }
+
+ YQL_ENSURE(false, "Unexpected provider class");
+ }
+
+ IGraphTransformer::TStatus ProcessDataProviderAnnotation(IDataProvider& dataProvider,
+ const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ auto status = static_cast<TDerived*>(this)->GetTransformer(dataProvider).Transform(input, output, ctx);
+ if (status.Level == IGraphTransformer::TStatus::Async) {
+ if (InstantOnly) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() <<
- "Async status is not allowed for instant transform, provider name: " << dataProvider.GetName()));
- return IGraphTransformer::TStatus::Error;
- }
-
- PendingNodes[input.Get()] = std::make_pair(input, &dataProvider);
- }
-
- return status;
- }
-
-protected:
- TTypeAnnotationContext& Types;
- const bool InstantOnly;
- TNodeMap<std::pair<TExprNode::TPtr, IDataProvider*>> PendingNodes;
-};
-
-} // NYql
+ "Async status is not allowed for instant transform, provider name: " << dataProvider.GetName()));
+ return IGraphTransformer::TStatus::Error;
+ }
+
+ PendingNodes[input.Get()] = std::make_pair(input, &dataProvider);
+ }
+
+ return status;
+ }
+
+protected:
+ TTypeAnnotationContext& Types;
+ const bool InstantOnly;
+ TNodeMap<std::pair<TExprNode::TPtr, IDataProvider*>> PendingNodes;
+};
+
+} // NYql
diff --git a/ydb/library/yql/core/yql_csv.cpp b/ydb/library/yql/core/yql_csv.cpp
index 259b6144f1..e620d15b02 100644
--- a/ydb/library/yql/core/yql_csv.cpp
+++ b/ydb/library/yql/core/yql_csv.cpp
@@ -1,4 +1,4 @@
-#include "yql_csv.h"
+#include "yql_csv.h"
#include <util/string/split.h>
#include <util/string/escape.h>
@@ -186,9 +186,9 @@ void TCsvOutputStream::DoWrite(const void* buf, size_t len)
}
}
-void TCsvOutputStream::DoFlush() {
- Slave_.Flush();
-}
-
+void TCsvOutputStream::DoFlush() {
+ Slave_.Flush();
+}
+
} // namspace NUtils
} // namspace NYql
diff --git a/ydb/library/yql/core/yql_csv.h b/ydb/library/yql/core/yql_csv.h
index 6e3e48b816..80cc6501ca 100644
--- a/ydb/library/yql/core/yql_csv.h
+++ b/ydb/library/yql/core/yql_csv.h
@@ -68,7 +68,7 @@ public:
private:
void DoWrite(const void* buf, size_t len) override final;
- void DoFlush() override;
+ void DoFlush() override;
private:
IOutputStream& Slave_;
diff --git a/ydb/library/yql/core/yql_data_provider.h b/ydb/library/yql/core/yql_data_provider.h
index 3ffbdc041d..877f239c73 100644
--- a/ydb/library/yql/core/yql_data_provider.h
+++ b/ydb/library/yql/core/yql_data_provider.h
@@ -1,7 +1,7 @@
#pragma once
#include "yql_graph_transformer.h"
-#include "yql_callable_names.h"
+#include "yql_callable_names.h"
#include <ydb/library/yql/public/udf/udf_validate.h>
@@ -9,8 +9,8 @@
#include <util/generic/maybe.h>
#include <util/generic/set.h>
-#include <util/generic/hash_set.h>
-#include <util/generic/string.h>
+#include <util/generic/hash_set.h>
+#include <util/generic/string.h>
#include <functional>
@@ -49,7 +49,7 @@ public:
virtual void WriteDetails(const TExprNode& node, NYson::TYsonWriter& writer) = 0;
// returns visibility of node
virtual bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) = 0;
- virtual void GetResultDependencies(const TExprNode::TPtr& node, TExprNode::TListType& children, bool compact) = 0;
+ virtual void GetResultDependencies(const TExprNode::TPtr& node, TExprNode::TListType& children, bool compact) = 0;
virtual void GetInputs(const TExprNode& node, TVector<TPinInfo>& inputs) = 0;
virtual void GetOutputs(const TExprNode& node, TVector<TPinInfo>& outputs) = 0;
virtual TString GetProviderPath(const TExprNode& node) = 0;
@@ -74,8 +74,8 @@ public:
virtual IGraphTransformer& GetCleanupTransformer() = 0;
};
-class IDqIntegration;
-
+class IDqIntegration;
+
class IOptimizationContext;
class IDataProvider : public TThrRefBase {
@@ -108,23 +108,23 @@ public:
//-- discovery & rewrite
virtual IGraphTransformer& GetIODiscoveryTransformer() = 0;
- //-- assign epochs
- virtual IGraphTransformer& GetEpochsTransformer() = 0;
-
+ //-- assign epochs
+ virtual IGraphTransformer& GetEpochsTransformer() = 0;
+
//-- intent determination
virtual IGraphTransformer& GetIntentDeterminationTransformer() = 0;
//-- type check
- virtual bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) = 0;
+ virtual bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) = 0;
virtual bool CanParse(const TExprNode& node) = 0;
virtual IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) = 0;
- virtual IGraphTransformer& GetConstraintTransformer(bool instantOnly, bool subGraph) = 0;
- // Fill set of callables, which have world as first child and should be trimmed in evaluation
- virtual void FillModifyCallables(THashSet<TStringBuf>& callables) = 0;
+ virtual IGraphTransformer& GetConstraintTransformer(bool instantOnly, bool subGraph) = 0;
+ // Fill set of callables, which have world as first child and should be trimmed in evaluation
+ virtual void FillModifyCallables(THashSet<TStringBuf>& callables) = 0;
//-- optimizations
virtual TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) = 0;
- virtual IGraphTransformer& GetRecaptureOptProposalTransformer() = 0;
+ virtual IGraphTransformer& GetRecaptureOptProposalTransformer() = 0;
virtual IGraphTransformer& GetLogicalOptProposalTransformer() = 0;
virtual IGraphTransformer& GetPhysicalOptProposalTransformer() = 0;
virtual IGraphTransformer& GetPhysicalFinalizingTransformer() = 0;
@@ -134,9 +134,9 @@ public:
//-- metadata loading
virtual IGraphTransformer& GetLoadTableMetadataTransformer() = 0;
- // This function is used in core optimizers to check either the node can be used as input multiple times or not
- virtual bool IsPersistent(const TExprNode& node) = 0;
-
+ // This function is used in core optimizers to check either the node can be used as input multiple times or not
+ virtual bool IsPersistent(const TExprNode& node) = 0;
+
// Right! or worlds are written to syncList
virtual bool CanBuildResult(const TExprNode& node, TSyncMap& syncList) = 0;
virtual bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) = 0;
@@ -165,9 +165,9 @@ public:
//-- garbage collection
virtual ITrackableNodeProcessor& GetTrackableNodeProcessor() = 0;
-
- // DQ
- virtual IDqIntegration* GetDqIntegration() = 0;
+
+ // DQ
+ virtual IDqIntegration* GetDqIntegration() = 0;
};
struct IPipelineConfigurator;
@@ -189,10 +189,10 @@ enum class ESourceSyntax {
struct TDataProviderInfo {
using TFutureStatus = NThreading::TFuture<IGraphTransformer::TStatus>;
- THashSet<TString> Names;
+ THashSet<TString> Names;
TIntrusivePtr<IDataProvider> Source;
TIntrusivePtr<IDataProvider> Sink;
- bool SupportFullResultDataSink = false;
+ bool SupportFullResultDataSink = false;
std::function<TMaybe<TString>(const TMaybe<TSet<TString>>& usedClusters, const TMaybe<TSet<TString>>& usedProviders,
ESourceSyntax syntax)> RemoteClusterProvider;
@@ -218,14 +218,14 @@ struct TDataProviderInfo {
std::function<NThreading::TFuture<void>(const TString& sessionId, const TString& username,
const TOperationProgressWriter& progressWriter, const TYqlOperationOptions& operationOptions,
- TIntrusivePtr<IRandomProvider> randomProvider, TIntrusivePtr<ITimeProvider> timeProvider)> OpenSession;
+ TIntrusivePtr<IRandomProvider> randomProvider, TIntrusivePtr<ITimeProvider> timeProvider)> OpenSession;
std::function<bool()> HasActiveProcesses;
std::function<void(const TString& sessionId)> CloseSession;
- std::function<void(const TString& sessionId)> CleanupSession;
-
+ std::function<void(const TString& sessionId)> CleanupSession;
+
std::function<TMaybe<TString>(const TString& url)> TokenResolver;
};
diff --git a/ydb/library/yql/core/yql_execution.cpp b/ydb/library/yql/core/yql_execution.cpp
index eb138b1b4f..0bcdac82d4 100644
--- a/ydb/library/yql/core/yql_execution.cpp
+++ b/ydb/library/yql/core/yql_execution.cpp
@@ -22,7 +22,7 @@ public:
TAdaptiveLock Lock;
struct TItem : public TIntrusiveListItem<TItem> {
- TExprNode* Node = nullptr;
+ TExprNode* Node = nullptr;
IDataProvider* DataProvider = nullptr;
};
@@ -34,7 +34,7 @@ public:
using TStatePtr = TIntrusivePtr<TState>;
- TExecutionTransformer(TTypeAnnotationContext& types,
+ TExecutionTransformer(TTypeAnnotationContext& types,
TOperationProgressWriter writer,
bool withFinalize)
: Types(types)
@@ -70,14 +70,14 @@ public:
}
status = status.Combine(ExecuteNode(input, output, ctx, 0));
- for (auto node: FreshPendingNodes) {
+ for (auto node: FreshPendingNodes) {
if (TExprNode::EState::ExecutionPending == node->GetState()) {
node->SetState(TExprNode::EState::ConstrComplete);
- }
- }
- FreshPendingNodes.clear();
- if (!ReplaceNewNodes(output, ctx)) {
- return TStatus::Error;
+ }
+ }
+ FreshPendingNodes.clear();
+ if (!ReplaceNewNodes(output, ctx)) {
+ return TStatus::Error;
}
YQL_CLOG(INFO, CoreExecution) << "Finish, output #" << output->UniqueId() << ", status: " << status;
@@ -132,31 +132,31 @@ public:
combinedStatus = combinedStatus.Combine(status);
if (status.Level == TStatus::Error) {
item.Node->SetState(TExprNode::EState::Error);
- } else if (status.Level == TStatus::Repeat) {
- if (callableOutput != item.Node) {
- YQL_CLOG(INFO, CoreExecution) << "Rewrite node #" << item.Node->UniqueId() << " to #" << callableOutput->UniqueId()
- << " in ApplyAsyncChanges()";
- NewNodes[item.Node] = callableOutput;
+ } else if (status.Level == TStatus::Repeat) {
+ if (callableOutput != item.Node) {
+ YQL_CLOG(INFO, CoreExecution) << "Rewrite node #" << item.Node->UniqueId() << " to #" << callableOutput->UniqueId()
+ << " in ApplyAsyncChanges()";
+ NewNodes[item.Node] = callableOutput;
combinedStatus = combinedStatus.Combine(TStatus(TStatus::Repeat, true));
- FinishNode(item.DataProvider->GetName(), *item.Node, *callableOutput);
- }
+ FinishNode(item.DataProvider->GetName(), *item.Node, *callableOutput);
+ }
}
- if (callableOutput == item.Node) {
+ if (callableOutput == item.Node) {
YQL_CLOG(INFO, CoreExecution) << "State is " << item.Node->GetState()
- << " after apply async changes for node #" << item.Node->UniqueId();
- }
+ << " after apply async changes for node #" << item.Node->UniqueId();
+ }
if (item.Node->GetState() == TExprNode::EState::ExecutionComplete ||
item.Node->GetState() == TExprNode::EState::Error)
{
- FinishNode(item.DataProvider->GetName(), *item.Node, *callableOutput);
+ FinishNode(item.DataProvider->GetName(), *item.Node, *callableOutput);
}
}
- if (!ReplaceNewNodes(output, ctx)) {
- return TStatus::Error;
- }
-
+ if (!ReplaceNewNodes(output, ctx)) {
+ return TStatus::Error;
+ }
+
if (!completed.Empty() && combinedStatus.Level == TStatus::Ok) {
combinedStatus = TStatus::Repeat;
}
@@ -164,42 +164,42 @@ public:
return combinedStatus;
}
- bool ReplaceNewNodes(TExprNode::TPtr& output, TExprContext& ctx) {
- if (!NewNodes.empty()) {
+ bool ReplaceNewNodes(TExprNode::TPtr& output, TExprContext& ctx) {
+ if (!NewNodes.empty()) {
TOptimizeExprSettings settings(&Types);
- settings.VisitChanges = true;
- settings.VisitStarted = true;
- auto replaceStatus = OptimizeExpr(output, output, [&](const TExprNode::TPtr& input, TExprContext& ctx) -> TExprNode::TPtr {
- Y_UNUSED(ctx);
- const auto replace = NewNodes.find(input.Get());
- if (NewNodes.cend() != replace) {
- return replace->second;
- }
-
- return input;
- }, ctx, settings);
-
- if (!RewriteSanityCheck) {
- NewNodes.clear();
- }
-
- if (replaceStatus.Level == TStatus::Error) {
- return false;
- }
- }
-
- if (RewriteSanityCheck) {
- VisitExpr(output, [&](const TExprNode::TPtr& localInput) {
- if (NewNodes.cend() != NewNodes.find(localInput.Get())) {
- Cerr << "found old node: #" << localInput->UniqueId() << "\n" << output->Dump();
- YQL_ENSURE(false);
- }
- return true;
- });
- }
- return true;
- }
-
+ settings.VisitChanges = true;
+ settings.VisitStarted = true;
+ auto replaceStatus = OptimizeExpr(output, output, [&](const TExprNode::TPtr& input, TExprContext& ctx) -> TExprNode::TPtr {
+ Y_UNUSED(ctx);
+ const auto replace = NewNodes.find(input.Get());
+ if (NewNodes.cend() != replace) {
+ return replace->second;
+ }
+
+ return input;
+ }, ctx, settings);
+
+ if (!RewriteSanityCheck) {
+ NewNodes.clear();
+ }
+
+ if (replaceStatus.Level == TStatus::Error) {
+ return false;
+ }
+ }
+
+ if (RewriteSanityCheck) {
+ VisitExpr(output, [&](const TExprNode::TPtr& localInput) {
+ if (NewNodes.cend() != NewNodes.find(localInput.Get())) {
+ Cerr << "found old node: #" << localInput->UniqueId() << "\n" << output->Dump();
+ YQL_ENSURE(false);
+ }
+ return true;
+ });
+ }
+ return true;
+ }
+
void Rewind() override {
State = MakeIntrusive<TState>();
State->Promise = NThreading::NewPromise();
@@ -214,32 +214,32 @@ public:
TStatus ExecuteNode(const TExprNode::TPtr& node, TExprNode::TPtr& output, TExprContext& ctx, ui32 depth) {
output = node;
- bool changed = false;
+ bool changed = false;
const auto knownNode = NewNodes.find(node.Get());
if (NewNodes.cend() != knownNode) {
output = knownNode->second;
- changed = true;
+ changed = true;
}
switch (output->GetState()) {
case TExprNode::EState::Initial:
case TExprNode::EState::TypeInProgress:
case TExprNode::EState::TypePending:
- case TExprNode::EState::TypeComplete:
- case TExprNode::EState::ConstrInProgress:
- case TExprNode::EState::ConstrPending:
+ case TExprNode::EState::TypeComplete:
+ case TExprNode::EState::ConstrInProgress:
+ case TExprNode::EState::ConstrPending:
return TStatus(TStatus::Repeat, true);
case TExprNode::EState::ExecutionInProgress:
return TStatus::Async;
case TExprNode::EState::ExecutionPending:
return ExecuteChildren(output, output, ctx, depth + 1);
- case TExprNode::EState::ConstrComplete:
+ case TExprNode::EState::ConstrComplete:
case TExprNode::EState::ExecutionRequired:
break;
case TExprNode::EState::ExecutionComplete:
YQL_ENSURE(output->HasResult());
OnNodeExecutionComplete(output, ctx);
- return changed ? TStatus(TStatus::Repeat, true) : TStatus(TStatus::Ok);
+ return changed ? TStatus(TStatus::Repeat, true) : TStatus(TStatus::Ok);
case TExprNode::EState::Error:
return TStatus::Error;
default:
@@ -255,13 +255,13 @@ public:
output->SetState(TExprNode::EState::Error);
return TStatus::Error;
- case TExprNode::List:
+ case TExprNode::List:
case TExprNode::Callable:
{
auto prevOutput = output;
- auto status = output->Type() == TExprNode::Callable
- ? ExecuteCallable(output, output, ctx, depth)
- : ExecuteList(output, ctx);
+ auto status = output->Type() == TExprNode::Callable
+ ? ExecuteCallable(output, output, ctx, depth)
+ : ExecuteList(output, ctx);
if (status.Level == TStatus::Error) {
output->SetState(TExprNode::EState::Error);
} else if (status.Level == TStatus::Ok) {
@@ -273,17 +273,17 @@ public:
output->SetState(TExprNode::EState::ExecutionPending);
status = ExecuteChildren(output, output, ctx, depth + 1);
if (TExprNode::EState::ExecutionPending == output->GetState()) {
- FreshPendingNodes.push_back(output.Get());
- }
- if (status.Level != TStatus::Repeat) {
- return status;
+ FreshPendingNodes.push_back(output.Get());
+ }
+ if (status.Level != TStatus::Repeat) {
+ return status;
}
}
- if (output != prevOutput) {
+ if (output != prevOutput) {
YQL_CLOG(INFO, CoreExecution) << "Rewrite node #" << node->UniqueId() << " to #" << output->UniqueId();
NewNodes[node.Get()] = output;
- }
- return TStatus(TStatus::Repeat, output != prevOutput);
+ }
+ return TStatus(TStatus::Repeat, output != prevOutput);
} else if (status.Level == TStatus::Async) {
output->SetState(TExprNode::EState::ExecutionInProgress);
}
@@ -315,7 +315,7 @@ public:
} else if (child->GetState() == TExprNode::EState::ExecutionInProgress) {
combinedStatus = combinedStatus.Combine(TStatus::Async);
} else if (child->GetState() == TExprNode::EState::ExecutionPending) {
- combinedStatus = combinedStatus.Combine(TStatus::Repeat);
+ combinedStatus = combinedStatus.Combine(TStatus::Repeat);
}
newChildren.push_back(newChild);
if (newChild != child) {
@@ -339,20 +339,20 @@ public:
}
}
- TStatus ExecuteList(const TExprNode::TPtr& node, TExprContext& ctx) {
- IGraphTransformer::TStatus combinedStatus = IGraphTransformer::TStatus::Ok;
- for (ui32 i = 0; i < node->ChildrenSize(); ++i) {
+ TStatus ExecuteList(const TExprNode::TPtr& node, TExprContext& ctx) {
+ IGraphTransformer::TStatus combinedStatus = IGraphTransformer::TStatus::Ok;
+ for (ui32 i = 0; i < node->ChildrenSize(); ++i) {
combinedStatus = combinedStatus.Combine(RequireChild(*node, i));
- }
-
- if (combinedStatus.Level != IGraphTransformer::TStatus::Ok) {
- return combinedStatus;
- }
-
- node->SetResult(ctx.NewWorld(node->Pos()));
- return TStatus::Ok;
- }
-
+ }
+
+ if (combinedStatus.Level != IGraphTransformer::TStatus::Ok) {
+ return combinedStatus;
+ }
+
+ node->SetResult(ctx.NewWorld(node->Pos()));
+ return TStatus::Ok;
+ }
+
IDataProvider* GetDataProvider(const TExprNode& node) const {
IDataProvider* dataProvider = nullptr;
for (const auto& x : Types.DataSources) {
@@ -382,7 +382,7 @@ public:
}
auto category = node->Child(1)->Child(0)->Content();
- auto datasink = Types.DataSinkMap.FindPtr(category);
+ auto datasink = Types.DataSinkMap.FindPtr(category);
YQL_ENSURE(datasink);
output = node;
auto status = (*datasink)->GetCallableExecutionTransformer().Transform(node, output, ctx);
@@ -394,8 +394,8 @@ public:
if (output->GetState() == TExprNode::EState::ExecutionComplete ||
output->GetState() == TExprNode::EState::Error)
{
- StartNode(category, *node);
- FinishNode(category, *node, *output);
+ StartNode(category, *node);
+ FinishNode(category, *node, *output);
}
}
@@ -403,7 +403,7 @@ public:
}
if (node->Content() == SyncName) {
- return ExecuteList(node, ctx);
+ return ExecuteList(node, ctx);
}
if (node->Content() == LeftName) {
@@ -442,8 +442,8 @@ public:
if (output->GetState() == TExprNode::EState::ExecutionComplete ||
output->GetState() == TExprNode::EState::Error)
{
- StartNode(dataProvider->GetName(), *node);
- FinishNode(dataProvider->GetName(), *node, *output);
+ StartNode(dataProvider->GetName(), *node);
+ FinishNode(dataProvider->GetName(), *node, *output);
}
}
@@ -483,9 +483,9 @@ public:
}
void StartNode(TStringBuf category, const TExprNode& node) {
- auto publicId = Types.TranslateOperationId(node.UniqueId());
+ auto publicId = Types.TranslateOperationId(node.UniqueId());
if (publicId) {
- auto x = Progresses.insert({ *publicId,
+ auto x = Progresses.insert({ *publicId,
TOperationProgress(TString(category), *publicId, TOperationProgress::EState::Started) });
if (x.second) {
Writer(x.first->second);
@@ -493,24 +493,24 @@ public:
}
}
- void FinishNode(TStringBuf category, const TExprNode& node, const TExprNode& newNode) {
- auto publicId = Types.TranslateOperationId(node.UniqueId());
+ void FinishNode(TStringBuf category, const TExprNode& node, const TExprNode& newNode) {
+ auto publicId = Types.TranslateOperationId(node.UniqueId());
if (publicId) {
- if (newNode.UniqueId() != node.UniqueId()) {
- Types.NodeToOperationId[newNode.UniqueId()] = *publicId;
- }
-
- auto progIt = Progresses.find(*publicId);
- YQL_ENSURE(progIt != Progresses.end());
+ if (newNode.UniqueId() != node.UniqueId()) {
+ Types.NodeToOperationId[newNode.UniqueId()] = *publicId;
+ }
+ auto progIt = Progresses.find(*publicId);
+ YQL_ENSURE(progIt != Progresses.end());
+
auto newState = (node.GetState() == TExprNode::EState::ExecutionComplete)
? TOperationProgress::EState::Finished
: TOperationProgress::EState::Failed;
- if (progIt->second.State != newState) {
+ if (progIt->second.State != newState) {
TString stage = progIt->second.Stage.first;
progIt->second = TOperationProgress(TString(category), *publicId, newState, stage);
- Writer(progIt->second);
+ Writer(progIt->second);
}
}
}
@@ -625,13 +625,13 @@ public:
}
private:
- TTypeAnnotationContext& Types;
+ TTypeAnnotationContext& Types;
TOperationProgressWriter Writer;
const bool WithFinalize;
TStatePtr State;
TNodeOnNodeOwnedMap NewNodes;
TAutoPtr<IGraphTransformer> FinalizingTransformer;
- THashMap<ui32, TOperationProgress> Progresses;
+ THashMap<ui32, TOperationProgress> Progresses;
struct TTrackableNodeInfo
{
@@ -642,25 +642,25 @@ private:
THashMap<TString, TTrackableNodeInfo> TrackableNodes;
TNodeOnNodeOwnedMap CollectingNodes;
THashMap<ui64, TIntrusivePtr<IDataProvider>> ProvidersCache;
- TExprNode::TListType FreshPendingNodes;
+ TExprNode::TListType FreshPendingNodes;
};
IGraphTransformer::TStatus ValidateExecution(const TExprNode::TPtr& node, TExprContext& ctx, const TTypeAnnotationContext& types, TNodeSet& visited);
-IGraphTransformer::TStatus ValidateList(const TExprNode::TPtr& node, TExprContext& ctx, const TTypeAnnotationContext& types, TNodeSet& visited) {
- IGraphTransformer::TStatus combinedStatus = IGraphTransformer::TStatus::Ok;
- for (ui32 i = 0; i < node->ChildrenSize(); ++i) {
- combinedStatus = combinedStatus.Combine(ValidateExecution(node->ChildPtr(i), ctx, types, visited));
- }
-
- return combinedStatus;
-}
-
+IGraphTransformer::TStatus ValidateList(const TExprNode::TPtr& node, TExprContext& ctx, const TTypeAnnotationContext& types, TNodeSet& visited) {
+ IGraphTransformer::TStatus combinedStatus = IGraphTransformer::TStatus::Ok;
+ for (ui32 i = 0; i < node->ChildrenSize(); ++i) {
+ combinedStatus = combinedStatus.Combine(ValidateExecution(node->ChildPtr(i), ctx, types, visited));
+ }
+
+ return combinedStatus;
+}
+
IGraphTransformer::TStatus ValidateCallable(const TExprNode::TPtr& node, TExprContext& ctx, const TTypeAnnotationContext& types, TNodeSet& visited) {
using TStatus = IGraphTransformer::TStatus;
if (node->Content() == CommitName) {
- auto datasink = types.DataSinkMap.FindPtr(node->Child(1)->Child(0)->Content());
+ auto datasink = types.DataSinkMap.FindPtr(node->Child(1)->Child(0)->Content());
if (!datasink) {
ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Unknown datasink: " << node->Child(1)->Child(0)->Content()));
return TStatus::Error;
@@ -670,7 +670,7 @@ IGraphTransformer::TStatus ValidateCallable(const TExprNode::TPtr& node, TExprCo
}
if (node->Content() == SyncName) {
- return ValidateList(node, ctx, types, visited);
+ return ValidateList(node, ctx, types, visited);
}
if (node->Content() == LeftName) {
@@ -683,16 +683,16 @@ IGraphTransformer::TStatus ValidateCallable(const TExprNode::TPtr& node, TExprCo
IDataProvider* dataProvider = nullptr;
for (auto& x : types.DataSources) {
- if (x->CanExecute(*node)) {
- dataProvider = x.Get();
+ if (x->CanExecute(*node)) {
+ dataProvider = x.Get();
break;
}
}
if (!dataProvider) {
for (auto& x : types.DataSinks) {
- if (x->CanExecute(*node)) {
- dataProvider = x.Get();
+ if (x->CanExecute(*node)) {
+ dataProvider = x.Get();
break;
}
}
@@ -726,7 +726,7 @@ IGraphTransformer::TStatus ValidateExecution(const TExprNode::TPtr& node, TExprC
return TStatus::Ok;
}
- if (!node->GetTypeAnn()) {
+ if (!node->GetTypeAnn()) {
ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), "Node has no type annotation"));
return TStatus::Error;
}
@@ -740,9 +740,9 @@ IGraphTransformer::TStatus ValidateExecution(const TExprNode::TPtr& node, TExprC
ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to execute node with type: " << node->Type()));
return TStatus::Error;
- case TExprNode::List:
- return ValidateList(node, ctx, types, visited);
-
+ case TExprNode::List:
+ return ValidateList(node, ctx, types, visited);
+
case TExprNode::Callable:
if (visited.cend() != visited.find(node.Get())) {
return TStatus::Ok;
@@ -768,7 +768,7 @@ IGraphTransformer::TStatus ValidateExecution(const TExprNode::TPtr& node, TExprC
}
TAutoPtr<IGraphTransformer> CreateExecutionTransformer(
- TTypeAnnotationContext& types,
+ TTypeAnnotationContext& types,
TOperationProgressWriter writer,
bool withFinalize) {
return new TExecutionTransformer(types, writer, withFinalize);
@@ -794,7 +794,7 @@ TAutoPtr<IGraphTransformer> CreateCheckExecutionTransformer(const TTypeAnnotatio
auto funcCheckExecution = [&](const THashSet<TStringBuf>& notAllowList, bool collectCalcOverWindow, const TExprNode::TPtr& node) {
if (node->IsCallable("ErrorType")) {
hasErrors = true;
- const auto err = node->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TErrorExprType>()->GetError();
+ const auto err = node->GetTypeAnn()->Cast<TTypeExprType>()->GetType()->Cast<TErrorExprType>()->GetError();
if (added.insert(err).second) {
ctx.AddError(err);
}
@@ -836,7 +836,7 @@ TAutoPtr<IGraphTransformer> CreateCheckExecutionTransformer(const TTypeAnnotatio
}
}
- if (usageCount > 1 && !node->GetConstraint<TEmptyConstraintNode>()) {
+ if (usageCount > 1 && !node->GetConstraint<TEmptyConstraintNode>()) {
hasErrors = true;
const auto err = TIssue(ctx.GetPosition(node->Pos()), "Multiple stream clients");
if (added.insert(err).second) {
@@ -868,13 +868,13 @@ TAutoPtr<IGraphTransformer> CreateCheckExecutionTransformer(const TTypeAnnotatio
IGraphTransformer::TStatus RequireChild(const TExprNode& node, ui32 index) {
switch (node.Child(index)->GetState()) {
case TExprNode::EState::Error:
- case TExprNode::EState::ExecutionComplete:
+ case TExprNode::EState::ExecutionComplete:
return IGraphTransformer::TStatus::Ok;
- case TExprNode::EState::ExecutionInProgress:
- case TExprNode::EState::ExecutionPending:
+ case TExprNode::EState::ExecutionInProgress:
+ case TExprNode::EState::ExecutionPending:
return IGraphTransformer::TStatus::Repeat;
- default:
- break;
+ default:
+ break;
}
node.Child(index)->SetState(TExprNode::EState::ExecutionRequired);
diff --git a/ydb/library/yql/core/yql_execution.h b/ydb/library/yql/core/yql_execution.h
index cb38d913fa..6d72891cf9 100644
--- a/ydb/library/yql/core/yql_execution.h
+++ b/ydb/library/yql/core/yql_execution.h
@@ -70,22 +70,22 @@ namespace NYql {
TMaybe<i64> Max;
TMaybe<i64> Min;
TMaybe<i64> Avg;
- TMaybe<i64> Count;
+ TMaybe<i64> Count;
TMaybe<TString> Value;
- TEntry(TString name, TMaybe<i64> sum, TMaybe<i64> max, TMaybe<i64> min, TMaybe<i64> avg, TMaybe<i64> count)
- : Name(std::move(name))
- , Sum(std::move(sum))
- , Max(std::move(max))
- , Min(std::move(min))
- , Avg(std::move(avg))
- , Count(std::move(count))
+ TEntry(TString name, TMaybe<i64> sum, TMaybe<i64> max, TMaybe<i64> min, TMaybe<i64> avg, TMaybe<i64> count)
+ : Name(std::move(name))
+ , Sum(std::move(sum))
+ , Max(std::move(max))
+ , Min(std::move(min))
+ , Avg(std::move(avg))
+ , Count(std::move(count))
{
}
- TEntry(TString name, TString value)
- : Name(std::move(name))
- , Value(std::move(value))
+ TEntry(TString name, TString value)
+ : Name(std::move(name))
+ , Value(std::move(value))
{
}
};
diff --git a/ydb/library/yql/core/yql_expr_constraint.cpp b/ydb/library/yql/core/yql_expr_constraint.cpp
index 78c8f68776..d6694a2904 100644
--- a/ydb/library/yql/core/yql_expr_constraint.cpp
+++ b/ydb/library/yql/core/yql_expr_constraint.cpp
@@ -1,126 +1,126 @@
-#include "yql_expr_constraint.h"
-#include "yql_callable_transform.h"
-#include "yql_opt_utils.h"
-
+#include "yql_expr_constraint.h"
+#include "yql_callable_transform.h"
+#include "yql_opt_utils.h"
+
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/core/yql_type_helpers.h>
#include <ydb/library/yql/core/yql_join.h>
#include <ydb/library/yql/utils/log/profile.h>
-
-#include <util/generic/scope.h>
-#include <util/generic/maybe.h>
-#include <util/generic/hash.h>
-#include <util/generic/utility.h>
-#include <util/generic/algorithm.h>
-#include <util/string/builder.h>
-#include <util/string/type.h>
-
-
-namespace NYql {
-
-using namespace NNodes;
-
-namespace {
-
-template <size_t FromChild, class... Other>
-struct TApplyConstraintFromInput;
-
-template <size_t FromChild>
-struct TApplyConstraintFromInput<FromChild> {
- static void Do(const TExprNode::TPtr&) {
- }
-};
-
-template <size_t FromChild, class TConstraint, class... Other>
-struct TApplyConstraintFromInput<FromChild, TConstraint, Other...> {
- static void Do(const TExprNode::TPtr& input) {
- if (auto c = input->Child(FromChild)->GetConstraint<TConstraint>()) {
- input->AddConstraint(c);
- }
- TApplyConstraintFromInput<FromChild, Other...>::Do(input);
- }
-};
-
-template <class TConstraint>
-const TConstraint* MakeCommonConstraint(const TExprNode::TPtr& input, size_t from, TExprContext& ctx) {
- TVector<const TConstraintSet*> constraints;
- for (size_t i = from; i < input->ChildrenSize(); ++i) {
- constraints.push_back(&input->Child(i)->GetConstraintSet());
- }
- return TConstraint::MakeCommon(constraints, ctx);
-}
-
-template <class... Other>
-struct TApplyCommonConstraint;
-
-template <class TConstraint>
-struct TApplyCommonConstraint<TConstraint> {
- static void Do(const TExprNode::TPtr& input, const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
- if (auto c = TConstraint::MakeCommon(constraints, ctx)) {
- input->AddConstraint(c);
- }
- }
-};
-
-template <class TConstraint, class... Other>
-struct TApplyCommonConstraint<TConstraint, Other...> {
- static void Do(const TExprNode::TPtr& input, const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
- if (auto c = TConstraint::MakeCommon(constraints, ctx)) {
- input->AddConstraint(c);
- }
- TApplyCommonConstraint<Other...>::Do(input, constraints, ctx);
- }
-};
-
-class TCallableConstraintTransformer : public TCallableTransformerBase<TCallableConstraintTransformer> {
- using THandler = TStatus(TCallableConstraintTransformer::*)(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&) const;
-
-public:
- TCallableConstraintTransformer(TTypeAnnotationContext& types, bool instantOnly, bool subGraph)
- : TCallableTransformerBase<TCallableConstraintTransformer>(types, instantOnly)
- , SubGraph(subGraph)
- {
- Functions["Unordered"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
+
+#include <util/generic/scope.h>
+#include <util/generic/maybe.h>
+#include <util/generic/hash.h>
+#include <util/generic/utility.h>
+#include <util/generic/algorithm.h>
+#include <util/string/builder.h>
+#include <util/string/type.h>
+
+
+namespace NYql {
+
+using namespace NNodes;
+
+namespace {
+
+template <size_t FromChild, class... Other>
+struct TApplyConstraintFromInput;
+
+template <size_t FromChild>
+struct TApplyConstraintFromInput<FromChild> {
+ static void Do(const TExprNode::TPtr&) {
+ }
+};
+
+template <size_t FromChild, class TConstraint, class... Other>
+struct TApplyConstraintFromInput<FromChild, TConstraint, Other...> {
+ static void Do(const TExprNode::TPtr& input) {
+ if (auto c = input->Child(FromChild)->GetConstraint<TConstraint>()) {
+ input->AddConstraint(c);
+ }
+ TApplyConstraintFromInput<FromChild, Other...>::Do(input);
+ }
+};
+
+template <class TConstraint>
+const TConstraint* MakeCommonConstraint(const TExprNode::TPtr& input, size_t from, TExprContext& ctx) {
+ TVector<const TConstraintSet*> constraints;
+ for (size_t i = from; i < input->ChildrenSize(); ++i) {
+ constraints.push_back(&input->Child(i)->GetConstraintSet());
+ }
+ return TConstraint::MakeCommon(constraints, ctx);
+}
+
+template <class... Other>
+struct TApplyCommonConstraint;
+
+template <class TConstraint>
+struct TApplyCommonConstraint<TConstraint> {
+ static void Do(const TExprNode::TPtr& input, const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
+ if (auto c = TConstraint::MakeCommon(constraints, ctx)) {
+ input->AddConstraint(c);
+ }
+ }
+};
+
+template <class TConstraint, class... Other>
+struct TApplyCommonConstraint<TConstraint, Other...> {
+ static void Do(const TExprNode::TPtr& input, const TVector<const TConstraintSet*>& constraints, TExprContext& ctx) {
+ if (auto c = TConstraint::MakeCommon(constraints, ctx)) {
+ input->AddConstraint(c);
+ }
+ TApplyCommonConstraint<Other...>::Do(input, constraints, ctx);
+ }
+};
+
+class TCallableConstraintTransformer : public TCallableTransformerBase<TCallableConstraintTransformer> {
+ using THandler = TStatus(TCallableConstraintTransformer::*)(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&) const;
+
+public:
+ TCallableConstraintTransformer(TTypeAnnotationContext& types, bool instantOnly, bool subGraph)
+ : TCallableTransformerBase<TCallableConstraintTransformer>(types, instantOnly)
+ , SubGraph(subGraph)
+ {
+ Functions["Unordered"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
Functions["UnorderedSubquery"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
- Functions["Sort"] = &TCallableConstraintTransformer::SortWrap;
- Functions["AssumeSorted"] = &TCallableConstraintTransformer::AssumeSortedWrap;
- Functions["AssumeUnique"] = &TCallableConstraintTransformer::AssumeUniqueWrap;
+ Functions["Sort"] = &TCallableConstraintTransformer::SortWrap;
+ Functions["AssumeSorted"] = &TCallableConstraintTransformer::AssumeSortedWrap;
+ Functions["AssumeUnique"] = &TCallableConstraintTransformer::AssumeUniqueWrap;
Functions["AssumeColumnOrder"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["AssumeAllMembersNullableAtOnce"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["Top"] = &TCallableConstraintTransformer::TopWrap<false>;
- Functions["TopSort"] = &TCallableConstraintTransformer::TopWrap<true>;
- Functions["TakeWhile"] = &TCallableConstraintTransformer::FilterWrap<true>;
- Functions["SkipWhile"] = &TCallableConstraintTransformer::FilterWrap<true>;
- Functions["TakeWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>;
- Functions["SkipWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>;
+ Functions["AssumeAllMembersNullableAtOnce"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["Top"] = &TCallableConstraintTransformer::TopWrap<false>;
+ Functions["TopSort"] = &TCallableConstraintTransformer::TopWrap<true>;
+ Functions["TakeWhile"] = &TCallableConstraintTransformer::FilterWrap<true>;
+ Functions["SkipWhile"] = &TCallableConstraintTransformer::FilterWrap<true>;
+ Functions["TakeWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>;
+ Functions["SkipWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>;
Functions["WideTakeWhile"] = &TCallableConstraintTransformer::FilterWrap<true>;
Functions["WideSkipWhile"] = &TCallableConstraintTransformer::FilterWrap<true>;
Functions["WideTakeWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>;
Functions["WideSkipWhileInclusive"] = &TCallableConstraintTransformer::FilterWrap<true>;
- Functions["Iterator"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["ForwardList"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["LazyList"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["Iterator"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["ForwardList"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["LazyList"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
Functions["ToFlow"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
Functions["FromFlow"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["ToStream"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["ToSequence"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["Collect"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["FilterNullMembers"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode>;
- Functions["SkipNullMembers"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode>;
+ Functions["ToStream"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["ToSequence"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["Collect"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["FilterNullMembers"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode>;
+ Functions["SkipNullMembers"] = &TCallableConstraintTransformer::FromFirst<TSortedConstraintNode, TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode>;
Functions["FilterNullElements"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TVarIndexConstraintNode>;
Functions["SkipNullElements"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TVarIndexConstraintNode>;
- Functions["Right!"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["Cons!"] = &TCallableConstraintTransformer::CopyAllFrom<1>;
- Functions["ExtractMembers"] = &TCallableConstraintTransformer::ExtractMembersWrap;
- Functions["RemoveSystemMembers"] = &TCallableConstraintTransformer::RemovePrefixMembersWrap;
- Functions["RemovePrefixMembers"] = &TCallableConstraintTransformer::RemovePrefixMembersWrap;
- Functions["SelectMembers"] = &TCallableConstraintTransformer::SelectMembersWrap;
- Functions["CastStruct"] = &TCallableConstraintTransformer::SelectMembersWrap;
+ Functions["Right!"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["Cons!"] = &TCallableConstraintTransformer::CopyAllFrom<1>;
+ Functions["ExtractMembers"] = &TCallableConstraintTransformer::ExtractMembersWrap;
+ Functions["RemoveSystemMembers"] = &TCallableConstraintTransformer::RemovePrefixMembersWrap;
+ Functions["RemovePrefixMembers"] = &TCallableConstraintTransformer::RemovePrefixMembersWrap;
+ Functions["SelectMembers"] = &TCallableConstraintTransformer::SelectMembersWrap;
+ Functions["CastStruct"] = &TCallableConstraintTransformer::SelectMembersWrap;
Functions["SafeCast"] = &TCallableConstraintTransformer::SelectMembersWrap<true>;
Functions["StrictCast"] = &TCallableConstraintTransformer::SelectMembersWrap<true>;
- Functions["DivePrefixMembers"] = &TCallableConstraintTransformer::DivePrefixMembersWrap;
- Functions["OrderedFilter"] = &TCallableConstraintTransformer::FilterWrap<true>;
- Functions["Filter"] = &TCallableConstraintTransformer::FilterWrap<false>;
+ Functions["DivePrefixMembers"] = &TCallableConstraintTransformer::DivePrefixMembersWrap;
+ Functions["OrderedFilter"] = &TCallableConstraintTransformer::FilterWrap<true>;
+ Functions["Filter"] = &TCallableConstraintTransformer::FilterWrap<false>;
Functions["WideFilter"] = &TCallableConstraintTransformer::FilterWrap<true>;
Functions["OrderedMap"] = &TCallableConstraintTransformer::MapWrap<true, false>;
Functions["Map"] = &TCallableConstraintTransformer::MapWrap<false, false>;
@@ -135,301 +135,301 @@ public:
Functions["NarrowMultiMap"] = &TCallableConstraintTransformer::MapWrap<true, false, true, false>;
Functions["OrderedFlatMapToEquiJoin"] = &TCallableConstraintTransformer::MapWrap<true, true>;
Functions["FlatMapToEquiJoin"] = &TCallableConstraintTransformer::MapWrap<false, true>;
- Functions["OrderedLMap"] = &TCallableConstraintTransformer::LMapWrap<true>;
- Functions["LMap"] = &TCallableConstraintTransformer::LMapWrap<false>;
- Functions["Extract"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
- Functions["OrderedExtract"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
- Functions["OrderedExtend"] = &TCallableConstraintTransformer::OrderedExtendWrap;
- Functions["Extend"] = &TCallableConstraintTransformer::ExtendWrap<false>;
- Functions["UnionAll"] = &TCallableConstraintTransformer::ExtendWrap<false>;
- Functions["Merge"] = &TCallableConstraintTransformer::MergeWrap<false>;
- Functions["UnionMerge"] = &TCallableConstraintTransformer::MergeWrap<true>;
- Functions["Skip"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["Take"] = &TCallableConstraintTransformer::TakeWrap;
- Functions["Limit"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
- Functions["Member"] = &TCallableConstraintTransformer::MemberWrap;
- Functions["AsStruct"] = &TCallableConstraintTransformer::AsStructWrap;
- Functions["Just"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
- Functions["Unwrap"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode, TEmptyConstraintNode>;
- Functions["ToList"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["OrderedLMap"] = &TCallableConstraintTransformer::LMapWrap<true>;
+ Functions["LMap"] = &TCallableConstraintTransformer::LMapWrap<false>;
+ Functions["Extract"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
+ Functions["OrderedExtract"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
+ Functions["OrderedExtend"] = &TCallableConstraintTransformer::OrderedExtendWrap;
+ Functions["Extend"] = &TCallableConstraintTransformer::ExtendWrap<false>;
+ Functions["UnionAll"] = &TCallableConstraintTransformer::ExtendWrap<false>;
+ Functions["Merge"] = &TCallableConstraintTransformer::MergeWrap<false>;
+ Functions["UnionMerge"] = &TCallableConstraintTransformer::MergeWrap<true>;
+ Functions["Skip"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["Take"] = &TCallableConstraintTransformer::TakeWrap;
+ Functions["Limit"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
+ Functions["Member"] = &TCallableConstraintTransformer::MemberWrap;
+ Functions["AsStruct"] = &TCallableConstraintTransformer::AsStructWrap;
+ Functions["Just"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
+ Functions["Unwrap"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode, TEmptyConstraintNode>;
+ Functions["ToList"] = &TCallableConstraintTransformer::CopyAllFrom<0>;
Functions["ToOptional"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
- Functions["Head"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
- Functions["Last"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
- Functions["Reverse"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
- Functions["Replicate"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
- Functions["AddMember"] = &TCallableConstraintTransformer::AddMemberWrap;
- Functions["RemoveMember"] = &TCallableConstraintTransformer::RemoveMemberWrap;
- Functions["ForceRemoveMember"] = &TCallableConstraintTransformer::RemoveMemberWrap;
- Functions["ReplaceMember"] = &TCallableConstraintTransformer::ReplaceMemberWrap;
- Functions["AsList"] = &TCallableConstraintTransformer::ExtendWrap<true>;
- Functions["OptionalIf"] = &TCallableConstraintTransformer::FromSecond<TPassthroughConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
- Functions["ListIf"] = &TCallableConstraintTransformer::CopyAllFrom<1>;
- Functions["FlatListIf"] = &TCallableConstraintTransformer::CopyAllFrom<1>;
- Functions["FlatOptionalIf"] = &TCallableConstraintTransformer::CopyAllFrom<1>;
- Functions["EmptyIterator"] = &TCallableConstraintTransformer::FromEmpty;
- Functions["List"] = &TCallableConstraintTransformer::ListWrap;
+ Functions["Head"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
+ Functions["Last"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
+ Functions["Reverse"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
+ Functions["Replicate"] = &TCallableConstraintTransformer::FromFirst<TPassthroughConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
+ Functions["AddMember"] = &TCallableConstraintTransformer::AddMemberWrap;
+ Functions["RemoveMember"] = &TCallableConstraintTransformer::RemoveMemberWrap;
+ Functions["ForceRemoveMember"] = &TCallableConstraintTransformer::RemoveMemberWrap;
+ Functions["ReplaceMember"] = &TCallableConstraintTransformer::ReplaceMemberWrap;
+ Functions["AsList"] = &TCallableConstraintTransformer::ExtendWrap<true>;
+ Functions["OptionalIf"] = &TCallableConstraintTransformer::FromSecond<TPassthroughConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
+ Functions["ListIf"] = &TCallableConstraintTransformer::CopyAllFrom<1>;
+ Functions["FlatListIf"] = &TCallableConstraintTransformer::CopyAllFrom<1>;
+ Functions["FlatOptionalIf"] = &TCallableConstraintTransformer::CopyAllFrom<1>;
+ Functions["EmptyIterator"] = &TCallableConstraintTransformer::FromEmpty;
+ Functions["List"] = &TCallableConstraintTransformer::ListWrap;
Functions["Dict"] = &TCallableConstraintTransformer::DictWrap;
- Functions["EmptyList"] = &TCallableConstraintTransformer::FromEmpty;
- Functions["EmptyDict"] = &TCallableConstraintTransformer::FromEmpty;
- Functions["DictItems"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
- Functions["DictKeys"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
- Functions["DictPayloads"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
- Functions["DictFromKeys"] = &TCallableConstraintTransformer::DictFromKeysWrap;
+ Functions["EmptyList"] = &TCallableConstraintTransformer::FromEmpty;
+ Functions["EmptyDict"] = &TCallableConstraintTransformer::FromEmpty;
+ Functions["DictItems"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
+ Functions["DictKeys"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
+ Functions["DictPayloads"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
+ Functions["DictFromKeys"] = &TCallableConstraintTransformer::DictFromKeysWrap;
Functions["If"] = &TCallableConstraintTransformer::IfWrap;
- Functions["Nothing"] = &TCallableConstraintTransformer::FromEmpty;
- Functions["IfPresent"] = &TCallableConstraintTransformer::IfPresentWrap;
+ Functions["Nothing"] = &TCallableConstraintTransformer::FromEmpty;
+ Functions["IfPresent"] = &TCallableConstraintTransformer::IfPresentWrap;
Functions["Coalesce"] = &TCallableConstraintTransformer::CommonFromChildren<0, TSortedConstraintNode, TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>;
- Functions["CombineByKey"] = &TCallableConstraintTransformer::FromFinalLambda<TCoCombineByKey::idx_FinishHandlerLambda>;
- Functions["CombineCore"] = &TCallableConstraintTransformer::FromFinalLambda<TCoCombineCore::idx_FinishHandler>;
- Functions["PartitionByKey"] = &TCallableConstraintTransformer::PartitionByKeyWrap;
- Functions["PartitionsByKeys"] = &TCallableConstraintTransformer::PartitionByKeyWrap;
- Functions["GroupByKey"] = &TCallableConstraintTransformer::GroupByKeyWrap;
- Functions["Switch"] = &TCallableConstraintTransformer::SwitchWrap;
- Functions["Visit"] = &TCallableConstraintTransformer::VisitWrap;
- Functions["VariantItem"] = &TCallableConstraintTransformer::VariantItemWrap;
- Functions["Variant"] = &TCallableConstraintTransformer::VariantWrap;
- Functions["Guess"] = &TCallableConstraintTransformer::GuessWrap;
- Functions["Mux"] = &TCallableConstraintTransformer::MuxWrap;
- Functions["Nth"] = &TCallableConstraintTransformer::NthWrap;
- Functions["EquiJoin"] = &TCallableConstraintTransformer::EquiJoinWrap;
- Functions["MapJoinCore"] = &TCallableConstraintTransformer::MapJoinCoreWrap;
- Functions["CommonJoinCore"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
- Functions["ToDict"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
- Functions["FoldMap"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; // TODO: passthrough
- Functions["Fold1Map"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; // TODO: passthrough
+ Functions["CombineByKey"] = &TCallableConstraintTransformer::FromFinalLambda<TCoCombineByKey::idx_FinishHandlerLambda>;
+ Functions["CombineCore"] = &TCallableConstraintTransformer::FromFinalLambda<TCoCombineCore::idx_FinishHandler>;
+ Functions["PartitionByKey"] = &TCallableConstraintTransformer::PartitionByKeyWrap;
+ Functions["PartitionsByKeys"] = &TCallableConstraintTransformer::PartitionByKeyWrap;
+ Functions["GroupByKey"] = &TCallableConstraintTransformer::GroupByKeyWrap;
+ Functions["Switch"] = &TCallableConstraintTransformer::SwitchWrap;
+ Functions["Visit"] = &TCallableConstraintTransformer::VisitWrap;
+ Functions["VariantItem"] = &TCallableConstraintTransformer::VariantItemWrap;
+ Functions["Variant"] = &TCallableConstraintTransformer::VariantWrap;
+ Functions["Guess"] = &TCallableConstraintTransformer::GuessWrap;
+ Functions["Mux"] = &TCallableConstraintTransformer::MuxWrap;
+ Functions["Nth"] = &TCallableConstraintTransformer::NthWrap;
+ Functions["EquiJoin"] = &TCallableConstraintTransformer::EquiJoinWrap;
+ Functions["MapJoinCore"] = &TCallableConstraintTransformer::MapJoinCoreWrap;
+ Functions["CommonJoinCore"] = &TCallableConstraintTransformer::FromFirst<TEmptyConstraintNode>;
+ Functions["ToDict"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
+ Functions["FoldMap"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; // TODO: passthrough
+ Functions["Fold1Map"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; // TODO: passthrough
Functions["Chain1Map"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; // TODO: passthrough, sorted, unique
Functions["WideChain1Map"] = &TCallableConstraintTransformer::InheriteEmptyFromInput; // TODO: passthrough, sorted, unique
- Functions["IsKeySwitch"] = &TCallableConstraintTransformer::IsKeySwitchWrap;
- Functions["Condense"] = &TCallableConstraintTransformer::CondenseWrap;
- Functions["Condense1"] = &TCallableConstraintTransformer::CondenseWrap;
- Functions["Squeeze"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
- Functions["Squeeze1"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
- Functions["GroupingCore"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
+ Functions["IsKeySwitch"] = &TCallableConstraintTransformer::IsKeySwitchWrap;
+ Functions["Condense"] = &TCallableConstraintTransformer::CondenseWrap;
+ Functions["Condense1"] = &TCallableConstraintTransformer::CondenseWrap;
+ Functions["Squeeze"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
+ Functions["Squeeze1"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
+ Functions["GroupingCore"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
Functions["Chopper"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
Functions["WideChopper"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
Functions["WideCombiner"] = &TCallableConstraintTransformer::InheriteEmptyFromInput;
Functions["WideCondense1"] = &TCallableConstraintTransformer::WideCondense1Wrap;
- Functions["Aggregate"] = &TCallableConstraintTransformer::AggregateWrap;
- Functions["Fold"] = &TCallableConstraintTransformer::FoldWrap;
- Functions["Fold1"] = &TCallableConstraintTransformer::FoldWrap;
- }
-
- TMaybe<IGraphTransformer::TStatus> ProcessCore(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- if (auto func = Functions.FindPtr(input->Content())) {
- return (this->**func)(input, output, ctx);
- }
- return Nothing();
- }
-
+ Functions["Aggregate"] = &TCallableConstraintTransformer::AggregateWrap;
+ Functions["Fold"] = &TCallableConstraintTransformer::FoldWrap;
+ Functions["Fold1"] = &TCallableConstraintTransformer::FoldWrap;
+ }
+
+ TMaybe<IGraphTransformer::TStatus> ProcessCore(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ if (auto func = Functions.FindPtr(input->Content())) {
+ return (this->**func)(input, output, ctx);
+ }
+ return Nothing();
+ }
+
TMaybe<IGraphTransformer::TStatus> ProcessList(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
if (!input->ChildrenSize() || ETypeAnnotationKind::Tuple != input->GetTypeAnn()->GetKind())
return TStatus::Ok;
return AsTupleWrap(input, output, ctx);
}
- TStatus ProcessUnknown(const TExprNode::TPtr& input, TExprContext&) {
- return UpdateAllChildLambdasConstraints(*input);
- }
-
- TStatus ValidateProviderCommitResult(const TExprNode::TPtr&, TExprContext&) {
- return TStatus::Ok;
- }
-
- TStatus ValidateProviderReadResult(const TExprNode::TPtr&, TExprContext&) {
- return TStatus::Ok;
- }
-
- TStatus ValidateProviderWriteResult(const TExprNode::TPtr&, TExprContext&) {
- return TStatus::Ok;
- }
-
- TStatus ValidateProviderConfigureResult(const TExprNode::TPtr&, TExprContext&) {
- return TStatus::Ok;
- }
-
- IGraphTransformer& GetTransformer(IDataProvider& provider) const {
- return provider.GetConstraintTransformer(InstantOnly, SubGraph);
- }
-
-private:
- template <size_t Ndx>
- TStatus CopyAllFrom(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- Y_UNUSED(output);
- Y_UNUSED(ctx);
- input->CopyConstraints(*input->Child(Ndx));
- return TStatus::Ok;
- }
-
- template <class... TConstraints>
- TStatus FromFirst(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- Y_UNUSED(output);
- Y_UNUSED(ctx);
- TApplyConstraintFromInput<0, TConstraints...>::Do(input);
- return TStatus::Ok;
- }
-
- template <class... TConstraints>
- TStatus FromSecond(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- Y_UNUSED(output);
- Y_UNUSED(ctx);
- TApplyConstraintFromInput<1, TConstraints...>::Do(input);
- return TStatus::Ok;
- }
-
- template <size_t StartFromChild, class... TConstraints>
- TStatus CommonFromChildren(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- Y_UNUSED(output);
- TVector<const TConstraintSet*> constraints;
- for (size_t i = StartFromChild; i < input->ChildrenSize(); ++i) {
- constraints.push_back(&input->Child(i)->GetConstraintSet());
- }
- TApplyCommonConstraint<TConstraints...>::Do(input, constraints, ctx);
- return TStatus::Ok;
- }
-
- template <size_t StartFromChild>
- TStatus AllCommonFromChildren(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- Y_UNUSED(output);
- TVector<const TConstraintSet*> constraints;
- for (size_t i = StartFromChild; i < input->ChildrenSize(); ++i) {
- constraints.push_back(&input->Child(i)->GetConstraintSet());
- }
- TApplyCommonConstraint<TSortedConstraintNode
- , TUniqueConstraintNode
- , TPassthroughConstraintNode
- , TEmptyConstraintNode
- , TVarIndexConstraintNode
- , TMultiConstraintNode
- >::Do(input, constraints, ctx);
- return TStatus::Ok;
- }
-
- TStatus FromEmpty(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- return TStatus::Ok;
- }
-
- template <size_t LambdaIdx>
- TStatus FromFinalLambda(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- TStatus status = UpdateAllChildLambdasConstraints(*input);
- if (status != TStatus::Ok) {
- return status;
- }
-
- TApplyConstraintFromInput<LambdaIdx, TMultiConstraintNode, TEmptyConstraintNode>::Do(input);
- return FromFirst<TEmptyConstraintNode>(input, output, ctx);
- }
-
- TStatus InheriteEmptyFromInput(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- auto status = UpdateAllChildLambdasConstraints(*input);
- if (status != TStatus::Ok) {
- return status;
- }
- return FromFirst<TEmptyConstraintNode>(input, output, ctx);
- }
-
- TStatus SortWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- auto status = UpdateLambdaConstraints(*input->Child(2));
- if (status != TStatus::Ok) {
- return status;
- }
-
- if (auto sorted = DeduceSortConstraint(*input->Child(0), *input->Child(1), *input->Child(2), ctx)) {
- input->AddConstraint(sorted);
- }
-
- return FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
- }
-
- TStatus AssumeSortedWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- auto status = UpdateLambdaConstraints(*input->Child(2));
- if (status != TStatus::Ok) {
- return status;
- }
-
- if (auto assumeConstr = DeduceSortConstraint(*input->Child(0), *input->Child(1), *input->Child(2), ctx)) {
- input->AddConstraint(assumeConstr);
- }
- return FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
- }
-
- TStatus AssumeUniqueWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- TVector<TStringBuf> columns;
- for (auto column: input->Child(1)->Children()) {
- columns.push_back(column->Content());
- }
- input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(columns)));
- return FromFirst<TPassthroughConstraintNode, TSortedConstraintNode, TEmptyConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
- }
-
- template <bool UseSort>
- TStatus TopWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- auto status = UpdateLambdaConstraints(*input->Child(3));
- if (status != TStatus::Ok) {
- return status;
- }
-
- if (UseSort) {
- if (auto sorted = DeduceSortConstraint(*input->Child(0), *input->Child(2), *input->Child(3), ctx)) {
- input->AddConstraint(sorted);
- }
- }
-
- return FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
- }
-
+ TStatus ProcessUnknown(const TExprNode::TPtr& input, TExprContext&) {
+ return UpdateAllChildLambdasConstraints(*input);
+ }
+
+ TStatus ValidateProviderCommitResult(const TExprNode::TPtr&, TExprContext&) {
+ return TStatus::Ok;
+ }
+
+ TStatus ValidateProviderReadResult(const TExprNode::TPtr&, TExprContext&) {
+ return TStatus::Ok;
+ }
+
+ TStatus ValidateProviderWriteResult(const TExprNode::TPtr&, TExprContext&) {
+ return TStatus::Ok;
+ }
+
+ TStatus ValidateProviderConfigureResult(const TExprNode::TPtr&, TExprContext&) {
+ return TStatus::Ok;
+ }
+
+ IGraphTransformer& GetTransformer(IDataProvider& provider) const {
+ return provider.GetConstraintTransformer(InstantOnly, SubGraph);
+ }
+
+private:
+ template <size_t Ndx>
+ TStatus CopyAllFrom(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ Y_UNUSED(output);
+ Y_UNUSED(ctx);
+ input->CopyConstraints(*input->Child(Ndx));
+ return TStatus::Ok;
+ }
+
+ template <class... TConstraints>
+ TStatus FromFirst(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ Y_UNUSED(output);
+ Y_UNUSED(ctx);
+ TApplyConstraintFromInput<0, TConstraints...>::Do(input);
+ return TStatus::Ok;
+ }
+
+ template <class... TConstraints>
+ TStatus FromSecond(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ Y_UNUSED(output);
+ Y_UNUSED(ctx);
+ TApplyConstraintFromInput<1, TConstraints...>::Do(input);
+ return TStatus::Ok;
+ }
+
+ template <size_t StartFromChild, class... TConstraints>
+ TStatus CommonFromChildren(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ Y_UNUSED(output);
+ TVector<const TConstraintSet*> constraints;
+ for (size_t i = StartFromChild; i < input->ChildrenSize(); ++i) {
+ constraints.push_back(&input->Child(i)->GetConstraintSet());
+ }
+ TApplyCommonConstraint<TConstraints...>::Do(input, constraints, ctx);
+ return TStatus::Ok;
+ }
+
+ template <size_t StartFromChild>
+ TStatus AllCommonFromChildren(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ Y_UNUSED(output);
+ TVector<const TConstraintSet*> constraints;
+ for (size_t i = StartFromChild; i < input->ChildrenSize(); ++i) {
+ constraints.push_back(&input->Child(i)->GetConstraintSet());
+ }
+ TApplyCommonConstraint<TSortedConstraintNode
+ , TUniqueConstraintNode
+ , TPassthroughConstraintNode
+ , TEmptyConstraintNode
+ , TVarIndexConstraintNode
+ , TMultiConstraintNode
+ >::Do(input, constraints, ctx);
+ return TStatus::Ok;
+ }
+
+ TStatus FromEmpty(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ return TStatus::Ok;
+ }
+
+ template <size_t LambdaIdx>
+ TStatus FromFinalLambda(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ TStatus status = UpdateAllChildLambdasConstraints(*input);
+ if (status != TStatus::Ok) {
+ return status;
+ }
+
+ TApplyConstraintFromInput<LambdaIdx, TMultiConstraintNode, TEmptyConstraintNode>::Do(input);
+ return FromFirst<TEmptyConstraintNode>(input, output, ctx);
+ }
+
+ TStatus InheriteEmptyFromInput(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ auto status = UpdateAllChildLambdasConstraints(*input);
+ if (status != TStatus::Ok) {
+ return status;
+ }
+ return FromFirst<TEmptyConstraintNode>(input, output, ctx);
+ }
+
+ TStatus SortWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ auto status = UpdateLambdaConstraints(*input->Child(2));
+ if (status != TStatus::Ok) {
+ return status;
+ }
+
+ if (auto sorted = DeduceSortConstraint(*input->Child(0), *input->Child(1), *input->Child(2), ctx)) {
+ input->AddConstraint(sorted);
+ }
+
+ return FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
+ }
+
+ TStatus AssumeSortedWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ auto status = UpdateLambdaConstraints(*input->Child(2));
+ if (status != TStatus::Ok) {
+ return status;
+ }
+
+ if (auto assumeConstr = DeduceSortConstraint(*input->Child(0), *input->Child(1), *input->Child(2), ctx)) {
+ input->AddConstraint(assumeConstr);
+ }
+ return FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
+ }
+
+ TStatus AssumeUniqueWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ TVector<TStringBuf> columns;
+ for (auto column: input->Child(1)->Children()) {
+ columns.push_back(column->Content());
+ }
+ input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(columns)));
+ return FromFirst<TPassthroughConstraintNode, TSortedConstraintNode, TEmptyConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
+ }
+
+ template <bool UseSort>
+ TStatus TopWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ auto status = UpdateLambdaConstraints(*input->Child(3));
+ if (status != TStatus::Ok) {
+ return status;
+ }
+
+ if (UseSort) {
+ if (auto sorted = DeduceSortConstraint(*input->Child(0), *input->Child(2), *input->Child(3), ctx)) {
+ input->AddConstraint(sorted);
+ }
+ }
+
+ return FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
+ }
+
template <bool CheckMembersType = false>
- TStatus SelectMembersWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- const TTypeAnnotationNode* outItemType = input->GetTypeAnn();
- while (outItemType->GetKind() == ETypeAnnotationKind::Optional) {
- outItemType = outItemType->Cast<TOptionalExprType>()->GetItemType();
- }
- if (outItemType->GetKind() == ETypeAnnotationKind::Variant) {
- if (outItemType->Cast<TVariantExprType>()->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
- const auto outSize = outItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>()->GetSize();
-
- auto multi = input->Child(0)->GetConstraint<TMultiConstraintNode>();
- if (multi && multi->GetItems().back().first >= outSize) {
- TMultiConstraintNode::TMapType filteredItems;
- for (auto& item: multi->GetItems()) {
- if (item.first < outSize) {
- filteredItems.push_back(item);
- }
- }
- multi = filteredItems.empty() ? nullptr : ctx.MakeConstraint<TMultiConstraintNode>(std::move(filteredItems));
- }
- if (multi) {
- input->AddConstraint(multi);
- }
-
- auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>();
- if (varIndex && varIndex->GetIndexMapping().back().first >= outSize) {
- TVarIndexConstraintNode::TMapType filteredItems;
- for (auto& item: varIndex->GetIndexMapping()) {
- if (item.first < outSize) {
- filteredItems.push_back(item);
- }
- }
- varIndex = filteredItems.empty() ? nullptr : ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(filteredItems));
- }
- if (varIndex) {
- input->AddConstraint(varIndex);
- }
- }
- }
- else if (outItemType->GetKind() == ETypeAnnotationKind::Struct) {
- const TStructExprType* outStructType = outItemType->Cast<TStructExprType>();
-
+ TStatus SelectMembersWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ const TTypeAnnotationNode* outItemType = input->GetTypeAnn();
+ while (outItemType->GetKind() == ETypeAnnotationKind::Optional) {
+ outItemType = outItemType->Cast<TOptionalExprType>()->GetItemType();
+ }
+ if (outItemType->GetKind() == ETypeAnnotationKind::Variant) {
+ if (outItemType->Cast<TVariantExprType>()->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
+ const auto outSize = outItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>()->GetSize();
+
+ auto multi = input->Child(0)->GetConstraint<TMultiConstraintNode>();
+ if (multi && multi->GetItems().back().first >= outSize) {
+ TMultiConstraintNode::TMapType filteredItems;
+ for (auto& item: multi->GetItems()) {
+ if (item.first < outSize) {
+ filteredItems.push_back(item);
+ }
+ }
+ multi = filteredItems.empty() ? nullptr : ctx.MakeConstraint<TMultiConstraintNode>(std::move(filteredItems));
+ }
+ if (multi) {
+ input->AddConstraint(multi);
+ }
+
+ auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>();
+ if (varIndex && varIndex->GetIndexMapping().back().first >= outSize) {
+ TVarIndexConstraintNode::TMapType filteredItems;
+ for (auto& item: varIndex->GetIndexMapping()) {
+ if (item.first < outSize) {
+ filteredItems.push_back(item);
+ }
+ }
+ varIndex = filteredItems.empty() ? nullptr : ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(filteredItems));
+ }
+ if (varIndex) {
+ input->AddConstraint(varIndex);
+ }
+ }
+ }
+ else if (outItemType->GetKind() == ETypeAnnotationKind::Struct) {
+ const TStructExprType* outStructType = outItemType->Cast<TStructExprType>();
+
if (const auto passthrough = input->Head().GetConstraint<TPassthroughConstraintNode>()) {
- TPassthroughConstraintNode::TMapType filteredMapping;
+ TPassthroughConstraintNode::TMapType filteredMapping;
if constexpr (CheckMembersType) {
- const TTypeAnnotationNode* inItemType = input->Head().GetTypeAnn();
- while (inItemType->GetKind() == ETypeAnnotationKind::Optional) {
- inItemType = inItemType->Cast<TOptionalExprType>()->GetItemType();
- }
+ const TTypeAnnotationNode* inItemType = input->Head().GetTypeAnn();
+ while (inItemType->GetKind() == ETypeAnnotationKind::Optional) {
+ inItemType = inItemType->Cast<TOptionalExprType>()->GetItemType();
+ }
const auto inStructType = inItemType->Cast<TStructExprType>();
- const auto& inItems = inStructType->GetItems();
- const auto& outItems = outStructType->GetItems();
+ const auto& inItems = inStructType->GetItems();
+ const auto& outItems = outStructType->GetItems();
for (const auto& part : passthrough->GetColumnMapping()) {
TPassthroughConstraintNode::TPartType filtered;
filtered.reserve(part.second.size());
@@ -444,9 +444,9 @@ private:
if (!filtered.empty()) {
filteredMapping.emplace(part.first ? part.first : passthrough, std::move(filtered));
}
- }
- }
- } else {
+ }
+ }
+ } else {
for (const auto& part: passthrough->GetColumnMapping()) {
TPassthroughConstraintNode::TPartType filtered;
filtered.reserve(part.second.size());
@@ -454,31 +454,31 @@ private:
if (!item.first.empty() && outStructType->FindItem(item.first.front())) {
filtered.push_back(item);
}
- }
+ }
if (!filtered.empty()) {
filteredMapping.emplace(part.first ? part.first : passthrough, std::move(filtered));
}
- }
- }
- if (!filteredMapping.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(filteredMapping)));
- }
- }
-
- if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
- if (AllOf(uniq->GetColumns(), [outStructType](TStringBuf col) { return outStructType->FindItem(col).Defined(); } )) {
- input->AddConstraint(uniq);
- }
- }
- }
-
- return TStatus::Ok;
- }
-
- TStatus DivePrefixMembersWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ }
+ }
+ if (!filteredMapping.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(filteredMapping)));
+ }
+ }
+
+ if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
+ if (AllOf(uniq->GetColumns(), [outStructType](TStringBuf col) { return outStructType->FindItem(col).Defined(); } )) {
+ input->AddConstraint(uniq);
+ }
+ }
+ }
+
+ return TStatus::Ok;
+ }
+
+ TStatus DivePrefixMembersWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
const auto prefixes = input->Child(1)->Children();
if (const auto passthrough = input->Head().GetConstraint<TPassthroughConstraintNode>()) {
- TPassthroughConstraintNode::TMapType filteredMapping;
+ TPassthroughConstraintNode::TMapType filteredMapping;
for (const auto& part: passthrough->GetColumnMapping()) {
TPassthroughConstraintNode::TPartType filtered;
filtered.reserve(part.second.size());
@@ -489,40 +489,40 @@ private:
filtered.insert_unique(std::move(item));
break;
}
- }
- }
+ }
+ }
if (!filtered.empty()) {
filteredMapping.emplace(part.first ? part.first : passthrough, std::move(filtered));
}
- }
- if (!filteredMapping.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(filteredMapping)));
- }
- }
-
- if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
- TUniqueConstraintNode::TSetType renamedUniq;
- for (auto& column: uniq->GetColumns()) {
- for (const auto& p : prefixes) {
- auto prefix = p->Content();
- if (column.StartsWith(prefix)) {
- renamedUniq.insert_unique(column.substr(prefix.length()));
- break;
- }
- }
- }
- if (!renamedUniq.empty() && renamedUniq.size() == uniq->GetColumns().size()) {
- input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(renamedUniq)));
- }
- }
-
- return FromFirst<TVarIndexConstraintNode>(input, output, ctx);
- }
-
- TStatus ExtractMembersWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- const TStructExprType* outItemType = GetItemType(*input->GetTypeAnn())->Cast<TStructExprType>();
+ }
+ if (!filteredMapping.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(filteredMapping)));
+ }
+ }
+
+ if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
+ TUniqueConstraintNode::TSetType renamedUniq;
+ for (auto& column: uniq->GetColumns()) {
+ for (const auto& p : prefixes) {
+ auto prefix = p->Content();
+ if (column.StartsWith(prefix)) {
+ renamedUniq.insert_unique(column.substr(prefix.length()));
+ break;
+ }
+ }
+ }
+ if (!renamedUniq.empty() && renamedUniq.size() == uniq->GetColumns().size()) {
+ input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(renamedUniq)));
+ }
+ }
+
+ return FromFirst<TVarIndexConstraintNode>(input, output, ctx);
+ }
+
+ TStatus ExtractMembersWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ const TStructExprType* outItemType = GetItemType(*input->GetTypeAnn())->Cast<TStructExprType>();
if (const auto passthrough = input->Head().GetConstraint<TPassthroughConstraintNode>()) {
- TPassthroughConstraintNode::TMapType filteredMapping;
+ TPassthroughConstraintNode::TMapType filteredMapping;
for (const auto& part: passthrough->GetColumnMapping()) {
TPassthroughConstraintNode::TPartType filtered;
filtered.reserve(part.second.size());
@@ -530,38 +530,38 @@ private:
if (!item.first.empty() && outItemType->FindItem(item.first.front())) {
filtered.push_back(item);
}
- }
+ }
if (!filtered.empty()) {
filteredMapping.emplace(part.first ? part.first : passthrough, std::move(filtered));
}
- }
- if (!filteredMapping.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(filteredMapping)));
- }
- }
-
+ }
+ if (!filteredMapping.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(filteredMapping)));
+ }
+ }
+
if (const auto sorted = TSortedConstraintNode::FilterByType(input->Head().GetConstraint<TSortedConstraintNode>(), outItemType, ctx))
input->AddConstraint(sorted);
-
- if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
- if (AllOf(uniq->GetColumns(), [outItemType](TStringBuf col) { return outItemType->FindItem(col).Defined(); } )) {
- input->AddConstraint(uniq);
- }
- }
-
- return FromFirst<TEmptyConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
- }
-
- TStatus RemovePrefixMembersWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- const TTypeAnnotationNode* outItemType = GetItemType(*input->GetTypeAnn());
- if (!outItemType) {
- outItemType = input->GetTypeAnn();
- }
-
- if (outItemType->GetKind() == ETypeAnnotationKind::Struct) {
+
+ if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
+ if (AllOf(uniq->GetColumns(), [outItemType](TStringBuf col) { return outItemType->FindItem(col).Defined(); } )) {
+ input->AddConstraint(uniq);
+ }
+ }
+
+ return FromFirst<TEmptyConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
+ }
+
+ TStatus RemovePrefixMembersWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ const TTypeAnnotationNode* outItemType = GetItemType(*input->GetTypeAnn());
+ if (!outItemType) {
+ outItemType = input->GetTypeAnn();
+ }
+
+ if (outItemType->GetKind() == ETypeAnnotationKind::Struct) {
const auto outStructType = outItemType->Cast<TStructExprType>();
if (const auto passthrough = input->Head().GetConstraint<TPassthroughConstraintNode>()) {
- TPassthroughConstraintNode::TMapType filteredMapping;
+ TPassthroughConstraintNode::TMapType filteredMapping;
for (const auto& part: passthrough->GetColumnMapping()) {
TPassthroughConstraintNode::TPartType filtered;
filtered.reserve(part.second.size());
@@ -569,37 +569,37 @@ private:
if (!item.first.empty() && outStructType->FindItem(item.first.front())) {
filtered.push_back(item);
}
- }
+ }
if (!filtered.empty()) {
filteredMapping.emplace(part.first ? part.first : passthrough, std::move(filtered));
}
- }
- if (!filteredMapping.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(filteredMapping)));
- }
- }
-
+ }
+ if (!filteredMapping.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(filteredMapping)));
+ }
+ }
+
if (const auto sorted = TSortedConstraintNode::FilterByType(input->Head().GetConstraint<TSortedConstraintNode>(), outStructType, ctx))
input->AddConstraint(sorted);
-
- if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
- if (AllOf(uniq->GetColumns(), [outStructType](TStringBuf col) { return outStructType->FindItem(col).Defined(); } )) {
- input->AddConstraint(uniq);
- }
- }
- }
- else if (outItemType->GetKind() == ETypeAnnotationKind::Variant) {
- if (auto multi = input->Child(0)->GetConstraint<TMultiConstraintNode>()) {
- TMultiConstraintNode::TMapType multiItems;
- auto tupleUnderType = outItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>();
- for (auto& item: multi->GetItems()) {
- YQL_ENSURE(item.first < tupleUnderType->GetSize());
-
- auto& constr = multiItems[item.first];
- const TStructExprType* outStructType = tupleUnderType->GetItems()[item.first]->Cast<TStructExprType>();
-
+
+ if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
+ if (AllOf(uniq->GetColumns(), [outStructType](TStringBuf col) { return outStructType->FindItem(col).Defined(); } )) {
+ input->AddConstraint(uniq);
+ }
+ }
+ }
+ else if (outItemType->GetKind() == ETypeAnnotationKind::Variant) {
+ if (auto multi = input->Child(0)->GetConstraint<TMultiConstraintNode>()) {
+ TMultiConstraintNode::TMapType multiItems;
+ auto tupleUnderType = outItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>();
+ for (auto& item: multi->GetItems()) {
+ YQL_ENSURE(item.first < tupleUnderType->GetSize());
+
+ auto& constr = multiItems[item.first];
+ const TStructExprType* outStructType = tupleUnderType->GetItems()[item.first]->Cast<TStructExprType>();
+
if (const auto passthrough = item.second.GetConstraint<TPassthroughConstraintNode>()) {
- TPassthroughConstraintNode::TMapType filteredMapping;
+ TPassthroughConstraintNode::TMapType filteredMapping;
for (const auto& part: passthrough->GetColumnMapping()) {
TPassthroughConstraintNode::TPartType filtered;
filtered.reserve(part.second.size());
@@ -607,46 +607,46 @@ private:
if (!item.first.empty() && outStructType->FindItem(item.first.front())) {
filtered.push_back(item);
}
- }
+ }
if (!filtered.empty()) {
filteredMapping.emplace(part.first ? part.first : passthrough, std::move(filtered));
}
- }
- if (!filteredMapping.empty()) {
- constr.AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(filteredMapping)));
- }
- }
-
+ }
+ if (!filteredMapping.empty()) {
+ constr.AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(filteredMapping)));
+ }
+ }
+
if (const auto sorted = TSortedConstraintNode::FilterByType(item.second.GetConstraint<TSortedConstraintNode>(), outStructType, ctx))
constr.AddConstraint(sorted);
-
- if (auto uniq = item.second.GetConstraint<TUniqueConstraintNode>()) {
- if (AllOf(uniq->GetColumns(), [outStructType](TStringBuf col) { return outStructType->FindItem(col).Defined(); } )) {
- constr.AddConstraint(uniq);
- }
- }
- }
- input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(std::move(multiItems)));
- }
- }
-
- return FromFirst<TEmptyConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
- }
-
- // TODO: Empty for false condition
- template <bool UseSorted>
- TStatus FilterWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+
+ if (auto uniq = item.second.GetConstraint<TUniqueConstraintNode>()) {
+ if (AllOf(uniq->GetColumns(), [outStructType](TStringBuf col) { return outStructType->FindItem(col).Defined(); } )) {
+ constr.AddConstraint(uniq);
+ }
+ }
+ }
+ input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(std::move(multiItems)));
+ }
+ }
+
+ return FromFirst<TEmptyConstraintNode, TVarIndexConstraintNode>(input, output, ctx);
+ }
+
+ // TODO: Empty for false condition
+ template <bool UseSorted>
+ TStatus FilterWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
if (const auto status = UpdateLambdaConstraints(*input->Child(1)); status != TStatus::Ok) {
- return status;
- }
-
+ return status;
+ }
+
if constexpr (UseSorted) {
- FromFirst<TSortedConstraintNode>(input, output, ctx);
- }
-
- return FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>(input, output, ctx);
- }
-
+ FromFirst<TSortedConstraintNode>(input, output, ctx);
+ }
+
+ return FromFirst<TPassthroughConstraintNode, TEmptyConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>(input, output, ctx);
+ }
+
template<class TConstraint>
static const TConstraint* GetConstraintFromWideResultLambda(const TExprNode& lambda, TExprContext& ctx);
@@ -743,35 +743,35 @@ private:
}
template <bool UseSorted, bool Flat, bool WideInput = false, bool WideOutput = false>
- TStatus MapWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ TStatus MapWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
const auto inItemType = GetItemType(*input->Child(0)->GetTypeAnn());
- bool multiInput = false;
+ bool multiInput = false;
TSmallVec<TConstraintNode::TListType> argConstraints(input->Tail().Head().ChildrenSize());
std::unordered_set<const TPassthroughConstraintNode*> explicitPasstrought;
- if (inItemType) {
+ if (inItemType) {
//TODO: Use GetConstraintsForInputArgument
- if (inItemType->GetKind() == ETypeAnnotationKind::Variant) {
- if (inItemType->Cast<TVariantExprType>()->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
+ if (inItemType->GetKind() == ETypeAnnotationKind::Variant) {
+ if (inItemType->Cast<TVariantExprType>()->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
const auto inputMulti = input->Head().GetConstraint<TMultiConstraintNode>();
const auto tupleType = inItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>();
argConstraints.front().push_back(ctx.MakeConstraint<TVarIndexConstraintNode>(*inItemType->Cast<TVariantExprType>()));
- TMultiConstraintNode::TMapType multiItems;
+ TMultiConstraintNode::TMapType multiItems;
multiItems.reserve(tupleType->GetSize());
-
- for (size_t i = 0; i < tupleType->GetSize(); ++i) {
+
+ for (size_t i = 0; i < tupleType->GetSize(); ++i) {
multiItems.emplace_back(i, TConstraintSet{});
const auto inputConstr = inputMulti ? inputMulti->GetItem(i) : nullptr;
- if (inputConstr) {
+ if (inputConstr) {
if (const auto empty = inputConstr->GetConstraint<TEmptyConstraintNode>()) {
- // TODO: multiItems.pop_back() instead
- multiItems.back().second.AddConstraint(empty);
- }
+ // TODO: multiItems.pop_back() instead
+ multiItems.back().second.AddConstraint(empty);
+ }
if (const auto pass = inputConstr->GetConstraint<TPassthroughConstraintNode>()) {
multiItems.back().second.AddConstraint(pass);
continue;
}
- }
+ }
switch (const auto variantItemType = tupleType->GetItems()[i]; variantItemType->GetKind()) {
/*TODO: case ETypeAnnotationKind::Tuple:
if (const auto argType = variantItemType->Cast<TTupleExprType>(); argType->GetSize() > 0U) {
@@ -791,13 +791,13 @@ private:
default:
break;
}
- }
- if (!multiItems.empty()) {
+ }
+ if (!multiItems.empty()) {
argConstraints.front().push_back(ctx.MakeConstraint<TMultiConstraintNode>(std::move(multiItems)));
- }
- multiInput = true;
- }
- } else {
+ }
+ multiInput = true;
+ }
+ } else {
const auto inputPassthrough = input->Head().GetConstraint<TPassthroughConstraintNode>();
if constexpr (WideInput) {
const auto structUnique = input->Head().GetConstraint<TUniqueConstraintNode>();
@@ -815,7 +815,7 @@ private:
argConstraints[i].emplace_back(fieldPasstrought);
}
}
- }
+ }
} else {
if (inputPassthrough)
argConstraints.front().emplace_back(inputPassthrough);
@@ -836,21 +836,21 @@ private:
if (const auto unique = input->Head().GetConstraint<TUniqueConstraintNode>()) {
argConstraints.front().push_back(unique);
- }
- }
+ }
+ }
if (const auto groupBy = input->Head().GetConstraint<TGroupByConstraintNode>()) {
argConstraints.front().push_back(groupBy);
- }
- }
- }
+ }
+ }
+ }
if (const auto status = UpdateLambdaConstraints(input->TailRef(), ctx, argConstraints); status != TStatus::Ok) {
- return status;
- }
-
- bool hasOutSorted = false;
+ return status;
+ }
+
+ bool hasOutSorted = false;
const auto lambdaPassthrough = GetConstraintFromLambda<TPassthroughConstraintNode, WideOutput>(input->Tail(), ctx);
- if (lambdaPassthrough) {
+ if (lambdaPassthrough) {
if (!explicitPasstrought.contains(lambdaPassthrough)) {
auto mapping = lambdaPassthrough->GetColumnMapping();
for (const auto myPasstrought : explicitPasstrought) {
@@ -859,57 +859,57 @@ private:
if (!mapping.empty()) {
input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(mapping)));
}
- }
-
+ }
+
if constexpr (UseSorted) {
if (const auto sorted = input->Head().GetConstraint<TSortedConstraintNode>()) {
if (const auto outSorted = GetPassthroughSortedConstraint(*sorted, *lambdaPassthrough, ctx)) {
- input->AddConstraint(outSorted);
- hasOutSorted = true;
- }
- }
- }
- }
-
+ input->AddConstraint(outSorted);
+ hasOutSorted = true;
+ }
+ }
+ }
+ }
+
if (input->Head().GetConstraint<TUniqueConstraintNode>()) {
if (const auto lambdaUnique = GetConstraintFromLambda<TUniqueConstraintNode, WideOutput>(input->Tail(), ctx)) {
- input->AddConstraint(lambdaUnique);
- }
- }
-
+ input->AddConstraint(lambdaUnique);
+ }
+ }
+
const auto lambdaVarIndex = GetConstraintFromLambda<TVarIndexConstraintNode, WideOutput>(input->Tail(), ctx);
const auto lambdaMulti = GetConstraintFromLambda<TMultiConstraintNode, WideOutput>(input->Tail(), ctx);
if (const auto varIndex = input->Head().GetConstraint<TVarIndexConstraintNode>()) {
- if (multiInput) {
- if (lambdaVarIndex) {
- if (auto outVarIndex = GetVarIndexOverVarIndexConstraint(*varIndex, *lambdaVarIndex, ctx)) {
- input->AddConstraint(outVarIndex);
- }
- }
- } else {
- if (lambdaMulti) {
- TVarIndexConstraintNode::TMapType remapItems;
- for (auto& multiItem: lambdaMulti->GetItems()) {
- for (auto& varItem: varIndex->GetIndexMapping()) {
- remapItems.push_back(std::make_pair(multiItem.first, varItem.second));
- }
- }
- if (!remapItems.empty()) {
- ::SortUnique(remapItems);
- input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(remapItems)));
- }
- } else {
- input->AddConstraint(varIndex);
- }
- }
- }
-
+ if (multiInput) {
+ if (lambdaVarIndex) {
+ if (auto outVarIndex = GetVarIndexOverVarIndexConstraint(*varIndex, *lambdaVarIndex, ctx)) {
+ input->AddConstraint(outVarIndex);
+ }
+ }
+ } else {
+ if (lambdaMulti) {
+ TVarIndexConstraintNode::TMapType remapItems;
+ for (auto& multiItem: lambdaMulti->GetItems()) {
+ for (auto& varItem: varIndex->GetIndexMapping()) {
+ remapItems.push_back(std::make_pair(multiItem.first, varItem.second));
+ }
+ }
+ if (!remapItems.empty()) {
+ ::SortUnique(remapItems);
+ input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(remapItems)));
+ }
+ } else {
+ input->AddConstraint(varIndex);
+ }
+ }
+ }
+
const auto inputMulti = input->Head().GetConstraint<TMultiConstraintNode>();
- if (lambdaMulti && !input->Child(0)->GetConstraint<TEmptyConstraintNode>()) {
- TMultiConstraintNode::TMapType remappedItems;
- for (auto& item: lambdaMulti->GetItems()) {
- remappedItems.push_back(std::make_pair(item.first, TConstraintSet{}));
- if (!multiInput) { // remapping one to many
+ if (lambdaMulti && !input->Child(0)->GetConstraint<TEmptyConstraintNode>()) {
+ TMultiConstraintNode::TMapType remappedItems;
+ for (auto& item: lambdaMulti->GetItems()) {
+ remappedItems.push_back(std::make_pair(item.first, TConstraintSet{}));
+ if (!multiInput) { // remapping one to many
if (const auto lambdaPassthrough = item.second.template GetConstraint<TPassthroughConstraintNode>()) {
if (!explicitPasstrought.contains(lambdaPassthrough)) {
auto mapping = lambdaPassthrough->GetColumnMapping();
@@ -919,31 +919,31 @@ private:
remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(mapping)));
}
}
-
+
if constexpr (UseSorted) {
if (const auto sorted = input->Child(0)->GetConstraint<TSortedConstraintNode>()) {
if (const auto outSorted = GetPassthroughSortedConstraint(*sorted, *lambdaPassthrough, ctx)) {
- remappedItems.back().second.AddConstraint(outSorted);
- }
- }
- }
- }
- if (input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
+ remappedItems.back().second.AddConstraint(outSorted);
+ }
+ }
+ }
+ }
+ if (input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
if (const auto lambdaUnique = item.second.template GetConstraint<TUniqueConstraintNode>()) {
- remappedItems.back().second.AddConstraint(lambdaUnique);
- }
- }
-
+ remappedItems.back().second.AddConstraint(lambdaUnique);
+ }
+ }
+
if (const auto empty = item.second.template GetConstraint<TEmptyConstraintNode>()) {
- remappedItems.pop_back();
- }
- }
+ remappedItems.pop_back();
+ }
+ }
else if (lambdaVarIndex && inputMulti) {
- const auto range = lambdaVarIndex->GetIndexMapping().equal_range(item.first);
- switch (std::distance(range.first, range.second)) {
- case 0: // new index
- break;
- case 1: // remapping 1 to 1
+ const auto range = lambdaVarIndex->GetIndexMapping().equal_range(item.first);
+ switch (std::distance(range.first, range.second)) {
+ case 0: // new index
+ break;
+ case 1: // remapping 1 to 1
if (const auto origConstr = inputMulti->GetItem(range.first->second)) {
if (const auto lambdaPassthrough = item.second.template GetConstraint<TPassthroughConstraintNode>()) {
if (!explicitPasstrought.contains(lambdaPassthrough)) {
@@ -953,218 +953,218 @@ private:
if (!mapping.empty()) {
remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(mapping)));
}
- }
-
+ }
+
if constexpr (UseSorted) {
if (const auto sorted = origConstr->template GetConstraint<TSortedConstraintNode>()) {
- if (auto outSorted = GetPassthroughSortedConstraint(*sorted, *lambdaPassthrough, ctx)) {
- remappedItems.back().second.AddConstraint(outSorted);
- }
- }
- }
- }
+ if (auto outSorted = GetPassthroughSortedConstraint(*sorted, *lambdaPassthrough, ctx)) {
+ remappedItems.back().second.AddConstraint(outSorted);
+ }
+ }
+ }
+ }
if (origConstr->template GetConstraint<TUniqueConstraintNode>()) {
if (const auto lambdaUnique = item.second.template GetConstraint<TUniqueConstraintNode>()) {
- remappedItems.back().second.AddConstraint(lambdaUnique);
- }
- }
+ remappedItems.back().second.AddConstraint(lambdaUnique);
+ }
+ }
if (const auto empty = item.second.template GetConstraint<TEmptyConstraintNode>()) {
- remappedItems.pop_back();
- }
- } else {
- remappedItems.pop_back();
- }
- break;
- default: // remapping many to one
- {
- std::vector<const TConstraintSet*> nonEmpty;
- for (auto i = range.first; i != range.second; ++i) {
+ remappedItems.pop_back();
+ }
+ } else {
+ remappedItems.pop_back();
+ }
+ break;
+ default: // remapping many to one
+ {
+ std::vector<const TConstraintSet*> nonEmpty;
+ for (auto i = range.first; i != range.second; ++i) {
if (auto origConstr = inputMulti->GetItem(i->second)) {
- nonEmpty.push_back(origConstr);
- }
- }
- EraseIf(nonEmpty, [] (const TConstraintSet* c) { return !!c->GetConstraint<TEmptyConstraintNode>(); });
-
- if (nonEmpty.empty()) {
- remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- } else if (nonEmpty.size() == 1) {
- remappedItems.back().second = std::move(*nonEmpty.front());
- }
- }
- }
- } else {
- remappedItems.back().second = item.second;
- }
- }
- if (remappedItems) {
- input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(std::move(remappedItems)));
- }
- }
+ nonEmpty.push_back(origConstr);
+ }
+ }
+ EraseIf(nonEmpty, [] (const TConstraintSet* c) { return !!c->GetConstraint<TEmptyConstraintNode>(); });
+
+ if (nonEmpty.empty()) {
+ remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ } else if (nonEmpty.size() == 1) {
+ remappedItems.back().second = std::move(*nonEmpty.front());
+ }
+ }
+ }
+ } else {
+ remappedItems.back().second = item.second;
+ }
+ }
+ if (remappedItems) {
+ input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(std::move(remappedItems)));
+ }
+ }
else if (inputMulti && lambdaVarIndex) { // Many to one
- const auto range = lambdaVarIndex->GetIndexMapping().equal_range(0);
- std::vector<const TConstraintSet*> nonEmpty;
- for (auto i = range.first; i != range.second; ++i) {
+ const auto range = lambdaVarIndex->GetIndexMapping().equal_range(0);
+ std::vector<const TConstraintSet*> nonEmpty;
+ for (auto i = range.first; i != range.second; ++i) {
if (auto origConstr = inputMulti->GetItem(i->second)) {
- nonEmpty.push_back(origConstr);
- }
- }
- EraseIf(nonEmpty, [] (const TConstraintSet* c) { return !!c->GetConstraint<TEmptyConstraintNode>(); });
-
- if (nonEmpty.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- } else if (nonEmpty.size() == 1) {
- input->SetConstraints(*nonEmpty.front());
- }
- }
-
+ nonEmpty.push_back(origConstr);
+ }
+ }
+ EraseIf(nonEmpty, [] (const TConstraintSet* c) { return !!c->GetConstraint<TEmptyConstraintNode>(); });
+
+ if (nonEmpty.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ } else if (nonEmpty.size() == 1) {
+ input->SetConstraints(*nonEmpty.front());
+ }
+ }
+
if (const auto lambdaEmpty = GetConstraintFromLambda<TEmptyConstraintNode, WideOutput>(input->Tail(), ctx)) {
- if (TCoFlatMapBase::Match(input.Get())) {
- input->AddConstraint(lambdaEmpty);
- }
- if (UseSorted && !hasOutSorted && !lambdaPassthrough) {
+ if (TCoFlatMapBase::Match(input.Get())) {
+ input->AddConstraint(lambdaEmpty);
+ }
+ if (UseSorted && !hasOutSorted && !lambdaPassthrough) {
if (const auto sorted = input->Child(0)->GetConstraint<TSortedConstraintNode>()) {
const auto outItemType = GetItemType(*input->GetTypeAnn());
- if (outItemType && outItemType->GetKind() == ETypeAnnotationKind::Struct) {
+ if (outItemType && outItemType->GetKind() == ETypeAnnotationKind::Struct) {
if (const auto outSorted = TSortedConstraintNode::FilterByType(sorted, outItemType->Cast<TStructExprType>(), ctx)) {
input->AddConstraint(outSorted);
- }
- }
- }
- }
- }
-
- return FromFirst<TEmptyConstraintNode>(input, output, ctx);
- }
-
- template <bool UseSorted>
- TStatus LMapWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- TConstraintNode::TListType argConstraints;
+ }
+ }
+ }
+ }
+ }
+
+ return FromFirst<TEmptyConstraintNode>(input, output, ctx);
+ }
+
+ template <bool UseSorted>
+ TStatus LMapWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ TConstraintNode::TListType argConstraints;
for (const auto c: input->Head().GetAllConstraints()) {
- if (UseSorted || c->GetName() != TSortedConstraintNode::Name()) {
- argConstraints.push_back(c);
- }
- }
-
+ if (UseSorted || c->GetName() != TSortedConstraintNode::Name()) {
+ argConstraints.push_back(c);
+ }
+ }
+
if (const auto status = UpdateLambdaConstraints(input->TailRef(), ctx, {argConstraints}); status != TStatus::Ok) {
return status;
- }
-
- TSet<TStringBuf> except;
- if (!UseSorted) {
- except.insert(TSortedConstraintNode::Name());
- }
+ }
+
+ TSet<TStringBuf> except;
+ if (!UseSorted) {
+ except.insert(TSortedConstraintNode::Name());
+ }
if (input->Tail().GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
- except.insert(TEmptyConstraintNode::Name());
- }
+ except.insert(TEmptyConstraintNode::Name());
+ }
CopyExcept(*input, input->Tail(), except);
-
- return FromFirst<TEmptyConstraintNode>(input, output, ctx);
- }
-
- template <bool AsList>
- TStatus ExtendWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- if (input->ChildrenSize() == 1) {
- if (auto unique = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
- input->AddConstraint(unique);
- }
- }
-
- if (AsList) {
- return CommonFromChildren<0, TPassthroughConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>(input, output, ctx);
- } else {
- return CommonFromChildren<0, TPassthroughConstraintNode, TEmptyConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>(input, output, ctx);
- }
- }
-
- template <bool Union>
- TStatus MergeWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- if (auto sort = MakeCommonConstraint<TSortedConstraintNode>(input, 0, ctx)) {
- if (Union && input->ChildrenSize() > 1) {
- // Check and exclude modified keys from final constraint
- auto resultStruct = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
- std::vector<const TStructExprType*> inputStructs;
- for (auto& child: input->Children()) {
- inputStructs.push_back(child->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>());
- }
+
+ return FromFirst<TEmptyConstraintNode>(input, output, ctx);
+ }
+
+ template <bool AsList>
+ TStatus ExtendWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ if (input->ChildrenSize() == 1) {
+ if (auto unique = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
+ input->AddConstraint(unique);
+ }
+ }
+
+ if (AsList) {
+ return CommonFromChildren<0, TPassthroughConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>(input, output, ctx);
+ } else {
+ return CommonFromChildren<0, TPassthroughConstraintNode, TEmptyConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>(input, output, ctx);
+ }
+ }
+
+ template <bool Union>
+ TStatus MergeWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ if (auto sort = MakeCommonConstraint<TSortedConstraintNode>(input, 0, ctx)) {
+ if (Union && input->ChildrenSize() > 1) {
+ // Check and exclude modified keys from final constraint
+ auto resultStruct = input->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>();
+ std::vector<const TStructExprType*> inputStructs;
+ for (auto& child: input->Children()) {
+ inputStructs.push_back(child->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>());
+ }
auto commonPrefixLength = sort->GetContent().size();
- for (size_t i = 0; i < commonPrefixLength; ++i) {
+ for (size_t i = 0; i < commonPrefixLength; ++i) {
auto column = sort->GetContent()[i].first.front();
- auto pos = resultStruct->FindItem(column);
+ auto pos = resultStruct->FindItem(column);
YQL_ENSURE(pos, "Missing column " << TString{column}.Quote() << " in result type");
- auto resultItemType = resultStruct->GetItems()[*pos];
- for (size_t childNdx = 0; childNdx < input->ChildrenSize(); ++childNdx) {
- const auto inputStruct = inputStructs[childNdx];
- if (auto pos = inputStruct->FindItem(column)) {
- if (resultItemType != inputStruct->GetItems()[*pos]) {
- commonPrefixLength = i;
- break;
- }
- } else {
- YQL_ENSURE(input->Child(childNdx)->GetConstraint<TEmptyConstraintNode>(), "Missing column " << TString{column}.Quote() << " in non empty input type");
- }
- }
- }
+ auto resultItemType = resultStruct->GetItems()[*pos];
+ for (size_t childNdx = 0; childNdx < input->ChildrenSize(); ++childNdx) {
+ const auto inputStruct = inputStructs[childNdx];
+ if (auto pos = inputStruct->FindItem(column)) {
+ if (resultItemType != inputStruct->GetItems()[*pos]) {
+ commonPrefixLength = i;
+ break;
+ }
+ } else {
+ YQL_ENSURE(input->Child(childNdx)->GetConstraint<TEmptyConstraintNode>(), "Missing column " << TString{column}.Quote() << " in non empty input type");
+ }
+ }
+ }
sort = sort->CutPrefix(commonPrefixLength, ctx);
- }
- if (sort) {
- input->AddConstraint(sort);
- }
- }
- return ExtendWrap<false>(input, output, ctx);
- }
-
- TStatus OrderedExtendWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- if (input->ChildrenSize() == 1) {
- if (auto sorted = input->Child(0)->GetConstraint<TSortedConstraintNode>()) {
- input->AddConstraint(sorted);
- }
- }
-
- return ExtendWrap<false>(input, output, ctx);
- }
-
- TStatus TakeWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- if (input->Tail().IsCallable("Uint64") && !FromString<ui64>(input->Tail().Head().Content())) {
+ }
+ if (sort) {
+ input->AddConstraint(sort);
+ }
+ }
+ return ExtendWrap<false>(input, output, ctx);
+ }
+
+ TStatus OrderedExtendWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ if (input->ChildrenSize() == 1) {
+ if (auto sorted = input->Child(0)->GetConstraint<TSortedConstraintNode>()) {
+ input->AddConstraint(sorted);
+ }
+ }
+
+ return ExtendWrap<false>(input, output, ctx);
+ }
+
+ TStatus TakeWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ if (input->Tail().IsCallable("Uint64") && !FromString<ui64>(input->Tail().Head().Content())) {
input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- }
+ }
return CopyAllFrom<0>(input, output, ctx);
- }
-
- TStatus MemberWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- TVector<TStringBuf> unique;
-
+ }
+
+ TStatus MemberWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ TVector<TStringBuf> unique;
+
const auto& memberName = input->Tail().Content();
const auto& structNode = input->Head();
if (const auto structPassthrough = structNode.GetConstraint<TPassthroughConstraintNode>()) {
if (const auto p = structPassthrough->ExtractField(ctx, memberName)) {
input->AddConstraint(p);
- }
- }
+ }
+ }
if (!TCoNothing::Match(&structNode)) {
if (auto structUnique = structNode.GetConstraint<TUniqueConstraintNode>()) {
- if (structUnique->GetColumns().has(memberName)) {
- unique.push_back(memberName);
- }
- }
- }
-
- if (!unique.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(unique)));
- }
-
+ if (structUnique->GetColumns().has(memberName)) {
+ unique.push_back(memberName);
+ }
+ }
+ }
+
+ if (!unique.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(unique)));
+ }
+
if (structNode.IsCallable("AsStruct")) {
for (auto& child: structNode.Children()) {
- if (child->Child(0)->Content() == memberName) {
- TApplyConstraintFromInput<1, TVarIndexConstraintNode>::Do(child);
- break;
- }
- }
- } else {
- TApplyConstraintFromInput<0, TVarIndexConstraintNode>::Do(input);
- }
- return TStatus::Ok;
- }
-
+ if (child->Child(0)->Content() == memberName) {
+ TApplyConstraintFromInput<1, TVarIndexConstraintNode>::Do(child);
+ break;
+ }
+ }
+ } else {
+ TApplyConstraintFromInput<0, TVarIndexConstraintNode>::Do(input);
+ }
+ return TStatus::Ok;
+ }
+
TStatus AsTupleWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
TPassthroughConstraintNode::TMapType passthrough;
passthrough.reserve(input->ChildrenSize());
@@ -1228,14 +1228,14 @@ private:
return TStatus::Ok;
}
- TStatus AsStructWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- TPassthroughConstraintNode::TMapType passthrough;
- passthrough.reserve(input->ChildrenSize());
+ TStatus AsStructWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ TPassthroughConstraintNode::TMapType passthrough;
+ passthrough.reserve(input->ChildrenSize());
- TVector<TStringBuf> unique;
- TVector<const TConstraintSet*> structConstraints;
+ TVector<TStringBuf> unique;
+ TVector<const TConstraintSet*> structConstraints;
TNodeSet srcStructs, arguments;
-
+
for (const auto& child : input->Children()) {
const auto& name = child->Head().Content();
if (const auto pass = child->Tail().GetConstraint<TPassthroughConstraintNode>()) {
@@ -1246,55 +1246,55 @@ private:
});
}
}
-
+
const auto valueNode = SkipModifiers(child->Child(1));
if (TCoMember::Match(valueNode) || TCoNth::Match(valueNode)) {
- auto memberName = valueNode->Child(1)->Content();
- auto structNode = valueNode->Child(0);
- if (auto structUnique = structNode->GetConstraint<TUniqueConstraintNode>()) {
- if (structUnique->GetColumns().has(memberName)) {
- if (!TCoNothing::Match(structNode)) {
- srcStructs.insert(structNode);
- unique.push_back(name);
- }
- }
- }
- structConstraints.push_back(&structNode->GetConstraintSet());
- }
- else if (valueNode->Type() == TExprNode::Argument) {
-
- if (auto structUnique = valueNode->GetConstraint<TUniqueConstraintNode>()) {
- if (structUnique->GetColumns().size() == 1) {
+ auto memberName = valueNode->Child(1)->Content();
+ auto structNode = valueNode->Child(0);
+ if (auto structUnique = structNode->GetConstraint<TUniqueConstraintNode>()) {
+ if (structUnique->GetColumns().has(memberName)) {
+ if (!TCoNothing::Match(structNode)) {
+ srcStructs.insert(structNode);
+ unique.push_back(name);
+ }
+ }
+ }
+ structConstraints.push_back(&structNode->GetConstraintSet());
+ }
+ else if (valueNode->Type() == TExprNode::Argument) {
+
+ if (auto structUnique = valueNode->GetConstraint<TUniqueConstraintNode>()) {
+ if (structUnique->GetColumns().size() == 1) {
arguments.insert(valueNode);
if (const auto scope = valueNode->GetDependencyScope())
srcStructs.insert(scope->second);
- unique.push_back(name);
- }
- }
- structConstraints.push_back(&valueNode->GetConstraintSet());
- }
- }
- if (!passthrough.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(passthrough)));
- }
- if (!unique.empty()) {
+ unique.push_back(name);
+ }
+ }
+ structConstraints.push_back(&valueNode->GetConstraintSet());
+ }
+ }
+ if (!passthrough.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(passthrough)));
+ }
+ if (!unique.empty()) {
if (srcStructs.empty() || (srcStructs.size() == 1 && (unique.size() == arguments.size() ||
(!(*srcStructs.begin())->IsLambda() && unique.size() == (*srcStructs.begin())->GetConstraint<TUniqueConstraintNode>()->GetColumns().size())))) {
- ::SortUnique(unique);
- input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(unique)));
- }
- }
- if (auto varIndex = TVarIndexConstraintNode::MakeCommon(structConstraints, ctx)) {
- input->AddConstraint(varIndex);
- }
-
- return TStatus::Ok;
- }
-
- TStatus AddMemberWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ ::SortUnique(unique);
+ input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(unique)));
+ }
+ }
+ if (auto varIndex = TVarIndexConstraintNode::MakeCommon(structConstraints, ctx)) {
+ input->AddConstraint(varIndex);
+ }
+
+ return TStatus::Ok;
+ }
+
+ TStatus AddMemberWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
const auto& addStructNode = input->Head();
const auto& name = input->Child(1)->Content();
-
+
if (const auto structPassthrough = addStructNode.GetConstraint<TPassthroughConstraintNode>(), fieldPasstrought = input->Tail().GetConstraint<TPassthroughConstraintNode>(); fieldPasstrought) {
auto mapping = structPassthrough ? structPassthrough->GetColumnMapping() : TPassthroughConstraintNode::TMapType();
if (const auto self = mapping.find(nullptr); mapping.cend() != self)
@@ -1309,59 +1309,59 @@ private:
input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(mapping)));
} else if (structPassthrough) {
input->AddConstraint(structPassthrough);
- }
-
+ }
+
auto valueNode = SkipModifiers(input->Child(2));
- TVector<const TConstraintSet*> structConstraints;
+ TVector<const TConstraintSet*> structConstraints;
structConstraints.push_back(&addStructNode.GetConstraintSet());
-
- TVector<TStringBuf> unique;
- TNodeSet srcStructs;
+
+ TVector<TStringBuf> unique;
+ TNodeSet srcStructs;
if (!TCoNothing::Match(&addStructNode)) {
if (auto structUnique = addStructNode.GetConstraint<TUniqueConstraintNode>()) {
srcStructs.insert(&addStructNode);
- unique.insert(unique.end(), structUnique->GetColumns().begin(), structUnique->GetColumns().end());
- }
- }
-
- if (TCoMember::Match(valueNode)) {
- auto memberName = valueNode->Child(1)->Content();
- auto structNode = valueNode->Child(0);
- if (auto structUnique = structNode->GetConstraint<TUniqueConstraintNode>()) {
- if (structUnique->GetColumns().has(memberName)) {
- if (!TCoNothing::Match(structNode)) {
- srcStructs.insert(structNode);
- unique.push_back(name);
- }
- }
- }
- structConstraints.push_back(&structNode->GetConstraintSet());
- }
- else if (valueNode->Type() == TExprNode::Argument) {
- if (auto structUnique = valueNode->GetConstraint<TUniqueConstraintNode>()) {
- if (structUnique->GetColumns().size() == 1) {
- srcStructs.insert(valueNode);
- unique.push_back(name);
- }
- }
- structConstraints.push_back(&valueNode->GetConstraintSet());
- }
-
- if (!unique.empty()) {
- if (srcStructs.empty() || (srcStructs.size() == 1 && unique.size() == (*srcStructs.begin())->GetConstraint<TUniqueConstraintNode>()->GetColumns().size())) {
- ::SortUnique(unique);
- input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(unique)));
- }
- }
-
- if (auto varIndex = TVarIndexConstraintNode::MakeCommon(structConstraints, ctx)) {
- input->AddConstraint(varIndex);
- }
- return TStatus::Ok;
- }
-
- TStatus RemoveMemberWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ unique.insert(unique.end(), structUnique->GetColumns().begin(), structUnique->GetColumns().end());
+ }
+ }
+
+ if (TCoMember::Match(valueNode)) {
+ auto memberName = valueNode->Child(1)->Content();
+ auto structNode = valueNode->Child(0);
+ if (auto structUnique = structNode->GetConstraint<TUniqueConstraintNode>()) {
+ if (structUnique->GetColumns().has(memberName)) {
+ if (!TCoNothing::Match(structNode)) {
+ srcStructs.insert(structNode);
+ unique.push_back(name);
+ }
+ }
+ }
+ structConstraints.push_back(&structNode->GetConstraintSet());
+ }
+ else if (valueNode->Type() == TExprNode::Argument) {
+ if (auto structUnique = valueNode->GetConstraint<TUniqueConstraintNode>()) {
+ if (structUnique->GetColumns().size() == 1) {
+ srcStructs.insert(valueNode);
+ unique.push_back(name);
+ }
+ }
+ structConstraints.push_back(&valueNode->GetConstraintSet());
+ }
+
+ if (!unique.empty()) {
+ if (srcStructs.empty() || (srcStructs.size() == 1 && unique.size() == (*srcStructs.begin())->GetConstraint<TUniqueConstraintNode>()->GetColumns().size())) {
+ ::SortUnique(unique);
+ input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(unique)));
+ }
+ }
+
+ if (auto varIndex = TVarIndexConstraintNode::MakeCommon(structConstraints, ctx)) {
+ input->AddConstraint(varIndex);
+ }
+ return TStatus::Ok;
+ }
+
+ TStatus RemoveMemberWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
if (const auto structPassthrough = input->Head().GetConstraint<TPassthroughConstraintNode>()) {
const TPassthroughConstraintNode::TKeyType key(1U, input->Tail().Content());
auto mapping = structPassthrough->GetColumnMapping();
@@ -1375,28 +1375,28 @@ private:
mapping.erase(p++);
continue;
}
- }
+ }
++p;
- }
+ }
if (!mapping.empty()) {
input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(mapping)));
}
- }
-
- if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
- if (TCoNothing::Match(input->Child(0)) || !uniq->GetColumns().has(input->Child(1)->Content())) {
- input->AddConstraint(uniq);
- }
- }
-
- return FromFirst<TVarIndexConstraintNode>(input, output, ctx);
- }
-
- TStatus ReplaceMemberWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ }
+
+ if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
+ if (TCoNothing::Match(input->Child(0)) || !uniq->GetColumns().has(input->Child(1)->Content())) {
+ input->AddConstraint(uniq);
+ }
+ }
+
+ return FromFirst<TVarIndexConstraintNode>(input, output, ctx);
+ }
+
+ TStatus ReplaceMemberWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
const auto name = input->Child(1)->Content();
- TVector<const TConstraintSet*> structConstraints;
- structConstraints.push_back(&input->Child(0)->GetConstraintSet());
-
+ TVector<const TConstraintSet*> structConstraints;
+ structConstraints.push_back(&input->Child(0)->GetConstraintSet());
+
if (const auto structPassthrough = input->Head().GetConstraint<TPassthroughConstraintNode>(), fieldPasstrought = input->Tail().GetConstraint<TPassthroughConstraintNode>();
structPassthrough || fieldPasstrought) {
auto mapping = structPassthrough ? structPassthrough->GetColumnMapping() : TPassthroughConstraintNode::TMapType();
@@ -1410,10 +1410,10 @@ private:
if (p->second.empty()) {
mapping.erase(p++);
continue;
- }
- }
+ }
+ }
++p;
- }
+ }
if (fieldPasstrought) {
for (const auto& part : fieldPasstrought->GetColumnMapping()) {
for (auto item : part.second) {
@@ -1422,10 +1422,10 @@ private:
}
}
}
-
- if (!mapping.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(mapping)));
- }
+
+ if (!mapping.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(mapping)));
+ }
if (structPassthrough && fieldPasstrought) {
if (auto valueNode = SkipModifiers(input->Child(2)); TCoMember::Match(valueNode)) {
@@ -1433,33 +1433,33 @@ private:
structConstraints.push_back(&structNode->GetConstraintSet());
}
}
- }
-
- if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
- if (!uniq->GetColumns().has(input->Child(1)->Content())) {
- input->AddConstraint(uniq);
- }
- }
-
- if (auto varIndex = TVarIndexConstraintNode::MakeCommon(structConstraints, ctx)) {
- input->AddConstraint(varIndex);
- }
-
- return TStatus::Ok;
- }
-
- TStatus ListWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- if (input->ChildrenSize() == 1) {
- return FromEmpty(input, output, ctx);
- } else if (input->ChildrenSize() == 2) {
- return FromSecond<TPassthroughConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>(input, output, ctx);
- }
- else if (input->ChildrenSize() > 2) {
- return CommonFromChildren<1, TPassthroughConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>(input, output, ctx);
- }
- return TStatus::Ok;
- }
-
+ }
+
+ if (auto uniq = input->Child(0)->GetConstraint<TUniqueConstraintNode>()) {
+ if (!uniq->GetColumns().has(input->Child(1)->Content())) {
+ input->AddConstraint(uniq);
+ }
+ }
+
+ if (auto varIndex = TVarIndexConstraintNode::MakeCommon(structConstraints, ctx)) {
+ input->AddConstraint(varIndex);
+ }
+
+ return TStatus::Ok;
+ }
+
+ TStatus ListWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ if (input->ChildrenSize() == 1) {
+ return FromEmpty(input, output, ctx);
+ } else if (input->ChildrenSize() == 2) {
+ return FromSecond<TPassthroughConstraintNode, TUniqueConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>(input, output, ctx);
+ }
+ else if (input->ChildrenSize() > 2) {
+ return CommonFromChildren<1, TPassthroughConstraintNode, TVarIndexConstraintNode, TMultiConstraintNode>(input, output, ctx);
+ }
+ return TStatus::Ok;
+ }
+
TStatus DictWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
if (input->ChildrenSize() == 1) {
return FromEmpty(input, output, ctx);
@@ -1467,13 +1467,13 @@ private:
return TStatus::Ok;
}
- TStatus DictFromKeysWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- if (input->Child(1)->ChildrenSize() == 0) {
- input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- }
- return TStatus::Ok;
- }
-
+ TStatus DictFromKeysWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ if (input->Child(1)->ChildrenSize() == 0) {
+ input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ }
+ return TStatus::Ok;
+ }
+
TStatus IfWrap(const TExprNode::TPtr& input, TExprNode::TPtr&, TExprContext& ctx) const {
TVector<const TConstraintSet*> constraints;
constraints.reserve((input->ChildrenSize() << 1U) + 1U);
@@ -1485,7 +1485,7 @@ private:
constraints.emplace_back(&child->GetConstraintSet());
}
}
-
+
if (constraints.empty())
input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
else if (1U == constraints.size())
@@ -1513,7 +1513,7 @@ private:
if (const auto status = UpdateLambdaConstraints(input->ChildRef(lambdaIndex), ctx, constraints); status != TStatus::Ok) {
return status;
- }
+ }
if (std::any_of(optionals.cbegin(), optionals.cend(), [] (const TExprNode::TPtr& node) { return bool(node->GetConstraint<TEmptyConstraintNode>()); })) {
input->CopyConstraints(input->Tail());
@@ -1529,11 +1529,11 @@ private:
, TMultiConstraintNode
>::Do(input, both, ctx);
return TStatus::Ok;
- }
-
- TStatus SwitchWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
- TStatus status = TStatus::Ok;
- TDynBitMap outFromChildren; // children, from which take a multi constraint for output
+ }
+
+ TStatus SwitchWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ TStatus status = TStatus::Ok;
+ TDynBitMap outFromChildren; // children, from which take a multi constraint for output
if (const auto multi = input->Head().GetConstraint<TMultiConstraintNode>()) {
for (size_t i = 2; i < input->ChildrenSize(); ++i) {
TMultiConstraintNode::TMapType items;
@@ -1543,7 +1543,7 @@ private:
if (auto c = multi->GetItem(index)) {
items[lambdaInputIndex] = *c;
outFromChildren.Set(i + 1);
- }
+ }
++lambdaInputIndex;
}
TConstraintNode::TListType argConstraints;
@@ -1553,107 +1553,107 @@ private:
argConstraints.push_back(ctx.MakeConstraint<TVarIndexConstraintNode>(input->Child(i)->ChildrenSize()));
} else {
argConstraints = items.front().second.GetAllConstraints();
- }
- }
+ }
+ }
status = status.Combine(UpdateLambdaConstraints(input->ChildRef(++i), ctx, {argConstraints}));
- }
- } else {
+ }
+ } else {
const bool inVar = GetSeqItemType(input->Head().GetTypeAnn())->GetKind() == ETypeAnnotationKind::Variant;
const TSmallVec<TConstraintNode::TListType> argConstraints(1U, inVar ? TConstraintNode::TListType() : input->Head().GetAllConstraints());
for (size_t i = 3; i < input->ChildrenSize(); i += 2) {
status = status.Combine(UpdateLambdaConstraints(input->ChildRef(i), ctx, argConstraints));
- }
- outFromChildren.Set(0, input->ChildrenSize());
- }
-
- if (status != TStatus::Ok) {
- return status;
- }
-
+ }
+ outFromChildren.Set(0, input->ChildrenSize());
+ }
+
+ if (status != TStatus::Ok) {
+ return status;
+ }
+
const auto inputVarIndex = input->Head().GetConstraint<TVarIndexConstraintNode>();
const bool emptyInput = input->Head().GetConstraint<TEmptyConstraintNode>();
if (GetSeqItemType(input->GetTypeAnn())->GetKind() == ETypeAnnotationKind::Variant) {
- ui32 outIndexOffset = 0;
- TMultiConstraintNode::TMapType multiItems;
- TVarIndexConstraintNode::TMapType remapItems;
- bool emptyOut = true;
- for (size_t i = 2; i < input->ChildrenSize(); i += 2) {
- const auto lambda = input->Child(i + 1);
+ ui32 outIndexOffset = 0;
+ TMultiConstraintNode::TMapType multiItems;
+ TVarIndexConstraintNode::TMapType remapItems;
+ bool emptyOut = true;
+ for (size_t i = 2; i < input->ChildrenSize(); i += 2) {
+ const auto lambda = input->Child(i + 1);
const auto lambdaItemType = GetSeqItemType(lambda->GetTypeAnn());
-
- if (inputVarIndex) {
- if (auto varIndex = lambda->GetConstraint<TVarIndexConstraintNode>()) {
- for (auto& item: varIndex->GetIndexMapping()) {
- YQL_ENSURE(item.second < input->Child(i)->ChildrenSize());
- const auto srcIndex = FromString<size_t>(input->Child(i)->Child(item.second)->Content());
- remapItems.push_back(std::make_pair(outIndexOffset + item.first, srcIndex));
- }
- } else if (lambdaItemType->GetKind() == ETypeAnnotationKind::Variant && input->Child(i)->ChildrenSize() == 1) {
- const auto srcIndex = FromString<size_t>(input->Child(i)->Child(0)->Content());
- for (size_t j = 0; j < lambdaItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>()->GetSize(); ++j) {
- remapItems.push_back(std::make_pair(outIndexOffset + j, srcIndex));
- }
- } else if (lambdaItemType->GetKind() != ETypeAnnotationKind::Variant && input->Child(i)->ChildrenSize() > 1) {
- for (auto& child : input->Child(i)->Children()) {
- const auto srcIndex = FromString<size_t>(child->Content());
- remapItems.push_back(std::make_pair(outIndexOffset, srcIndex));
- }
- }
- }
-
- const bool lambdaEmpty = lambda->GetConstraint<TEmptyConstraintNode>();
- if (!lambdaEmpty) {
- emptyOut = false;
- }
- if (lambdaItemType->GetKind() == ETypeAnnotationKind::Variant) {
- if (!emptyInput && outFromChildren.Test(i + 1)) {
- if (auto multi = lambda->GetConstraint<TMultiConstraintNode>()) {
- for (auto& item: multi->GetItems()) {
- multiItems.insert_unique(std::make_pair(outIndexOffset + item.first, item.second));
- }
- }
- }
- outIndexOffset += lambdaItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>()->GetSize();
- } else {
- if (!emptyInput && outFromChildren.Test(i + 1) && !lambdaEmpty) {
- multiItems[outIndexOffset] = lambda->GetConstraintSet();
- }
- ++outIndexOffset;
- }
- }
-
- if (inputVarIndex && !remapItems.empty()) {
- TVarIndexConstraintNode::TMapType result;
- for (auto& item: remapItems) {
- auto range = inputVarIndex->GetIndexMapping().equal_range(item.second);
- for (auto it = range.first; it != range.second; ++it) {
- result.push_back(std::make_pair(item.first, it->second));
- }
- }
- if (!result.empty()) {
- ::Sort(result);
- input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(result)));
- }
- }
-
- if (!multiItems.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(std::move(multiItems)));
- }
- if (emptyOut) {
- input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- }
- } else {
- YQL_ENSURE(input->ChildrenSize() == 4);
- input->CopyConstraints(*input->Child(3));
- }
- return FromFirst<TEmptyConstraintNode>(input, output, ctx);
- }
-
+
+ if (inputVarIndex) {
+ if (auto varIndex = lambda->GetConstraint<TVarIndexConstraintNode>()) {
+ for (auto& item: varIndex->GetIndexMapping()) {
+ YQL_ENSURE(item.second < input->Child(i)->ChildrenSize());
+ const auto srcIndex = FromString<size_t>(input->Child(i)->Child(item.second)->Content());
+ remapItems.push_back(std::make_pair(outIndexOffset + item.first, srcIndex));
+ }
+ } else if (lambdaItemType->GetKind() == ETypeAnnotationKind::Variant && input->Child(i)->ChildrenSize() == 1) {
+ const auto srcIndex = FromString<size_t>(input->Child(i)->Child(0)->Content());
+ for (size_t j = 0; j < lambdaItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>()->GetSize(); ++j) {
+ remapItems.push_back(std::make_pair(outIndexOffset + j, srcIndex));
+ }
+ } else if (lambdaItemType->GetKind() != ETypeAnnotationKind::Variant && input->Child(i)->ChildrenSize() > 1) {
+ for (auto& child : input->Child(i)->Children()) {
+ const auto srcIndex = FromString<size_t>(child->Content());
+ remapItems.push_back(std::make_pair(outIndexOffset, srcIndex));
+ }
+ }
+ }
+
+ const bool lambdaEmpty = lambda->GetConstraint<TEmptyConstraintNode>();
+ if (!lambdaEmpty) {
+ emptyOut = false;
+ }
+ if (lambdaItemType->GetKind() == ETypeAnnotationKind::Variant) {
+ if (!emptyInput && outFromChildren.Test(i + 1)) {
+ if (auto multi = lambda->GetConstraint<TMultiConstraintNode>()) {
+ for (auto& item: multi->GetItems()) {
+ multiItems.insert_unique(std::make_pair(outIndexOffset + item.first, item.second));
+ }
+ }
+ }
+ outIndexOffset += lambdaItemType->Cast<TVariantExprType>()->GetUnderlyingType()->Cast<TTupleExprType>()->GetSize();
+ } else {
+ if (!emptyInput && outFromChildren.Test(i + 1) && !lambdaEmpty) {
+ multiItems[outIndexOffset] = lambda->GetConstraintSet();
+ }
+ ++outIndexOffset;
+ }
+ }
+
+ if (inputVarIndex && !remapItems.empty()) {
+ TVarIndexConstraintNode::TMapType result;
+ for (auto& item: remapItems) {
+ auto range = inputVarIndex->GetIndexMapping().equal_range(item.second);
+ for (auto it = range.first; it != range.second; ++it) {
+ result.push_back(std::make_pair(item.first, it->second));
+ }
+ }
+ if (!result.empty()) {
+ ::Sort(result);
+ input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(result)));
+ }
+ }
+
+ if (!multiItems.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(std::move(multiItems)));
+ }
+ if (emptyOut) {
+ input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ }
+ } else {
+ YQL_ENSURE(input->ChildrenSize() == 4);
+ input->CopyConstraints(*input->Child(3));
+ }
+ return FromFirst<TEmptyConstraintNode>(input, output, ctx);
+ }
+
TStatus VisitWrap(const TExprNode::TPtr& input, TExprNode::TPtr&, TExprContext& ctx) const {
- TStatus status = TStatus::Ok;
- TDynBitMap outFromChildren; // children, from which take a multi constraint for output
- TDynBitMap usedAlts;
+ TStatus status = TStatus::Ok;
+ TDynBitMap outFromChildren; // children, from which take a multi constraint for output
+ TDynBitMap usedAlts;
const auto inMulti = input->Head().GetConstraint<TMultiConstraintNode>();
for (ui32 i = 1; i < input->ChildrenSize(); ++i) {
if (const auto child = input->Child(i); child->IsAtom()) {
@@ -1664,8 +1664,8 @@ private:
if (const auto c = inMulti->GetItem(index)) {
argConstraints.front() = c->GetAllConstraints();
outFromChildren.Set(i + 1U);
- }
- }
+ }
+ }
status = status.Combine(UpdateLambdaConstraints(input->ChildRef(++i), ctx, argConstraints));
} else if (inMulti) { // Check that we can fall to default branch
for (auto& item: inMulti->GetItems()) {
@@ -1674,230 +1674,230 @@ private:
break;
}
}
- }
- }
-
- if (status != TStatus::Ok) {
- return status;
- }
+ }
+ }
+ if (status != TStatus::Ok) {
+ return status;
+ }
+
if (!inMulti) {
outFromChildren.Set(0, input->ChildrenSize());
}
- auto outType = input->GetTypeAnn();
- if (auto t = GetItemType(*outType)) {
- outType = t;
- }
- if (outType->GetKind() == ETypeAnnotationKind::Variant && outType->Cast<TVariantExprType>()->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
- TVector<const TConstraintSet*> outConstraints;
- TVarIndexConstraintNode::TMapType remapItems;
- for (ui32 i = 1; i < input->ChildrenSize(); ++i) {
- if (input->Child(i)->IsAtom()) {
- ui32 index = FromString<ui32>(input->Child(i)->Content());
- ++i;
- if (outFromChildren.Test(i)) {
- outConstraints.push_back(&input->Child(i)->GetConstraintSet());
- if (const auto outMulti = input->Child(i)->GetConstraint<TMultiConstraintNode>()) {
- for (auto& item: outMulti->GetItems()) {
- remapItems.push_back(std::make_pair(item.first, index));
- }
- }
- }
- } else {
- if (outFromChildren.Test(i)) {
- outConstraints.push_back(&input->Child(i)->GetConstraintSet());
- const auto outMulti = input->Child(i)->GetConstraint<TMultiConstraintNode>();
- if (outMulti && inMulti) {
- for (auto& outItem: outMulti->GetItems()) {
- for (auto& inItem: inMulti->GetItems()) {
- if (!usedAlts.Test(inItem.first)) {
- remapItems.push_back(std::make_pair(outItem.first, inItem.first));
- }
- }
- }
- }
- }
- }
- }
-
- if (auto multi = TMultiConstraintNode::MakeCommon(outConstraints, ctx)) {
- input->AddConstraint(multi);
- }
-
- if (auto empty = TEmptyConstraintNode::MakeCommon(outConstraints, ctx)) {
- input->AddConstraint(empty);
- }
-
- if (auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>()) {
- TVarIndexConstraintNode::TMapType varIndexItems;
- for (auto& item: remapItems) {
- const auto range = varIndex->GetIndexMapping().equal_range(item.second);
- for (auto i = range.first; i != range.second; ++i) {
- varIndexItems.push_back(std::make_pair(item.first, i->second));
- }
- }
- if (!varIndexItems.empty()) {
- ::Sort(varIndexItems);
- input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(varIndexItems)));
- }
- }
- }
- else {
- std::vector<const TConstraintSet*> nonEmpty;
- for (ui32 i = 1; i < input->ChildrenSize(); ++i) {
- if (input->Child(i)->IsAtom()) {
- ++i;
- }
- if (outFromChildren.Test(i)) {
- nonEmpty.push_back(&input->Child(i)->GetConstraintSet());
- }
- }
- EraseIf(nonEmpty, [] (const TConstraintSet* c) { return !!c->GetConstraint<TEmptyConstraintNode>(); });
-
- if (nonEmpty.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- } else if (nonEmpty.size() == 1) {
- input->SetConstraints(*nonEmpty.front());
- }
-
- if (auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>()) {
- TVarIndexConstraintNode::TMapType varIndexItems;
- for (ui32 i = 1; i < input->ChildrenSize(); ++i) {
- if (input->Child(i)->IsAtom()) {
+ auto outType = input->GetTypeAnn();
+ if (auto t = GetItemType(*outType)) {
+ outType = t;
+ }
+ if (outType->GetKind() == ETypeAnnotationKind::Variant && outType->Cast<TVariantExprType>()->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
+ TVector<const TConstraintSet*> outConstraints;
+ TVarIndexConstraintNode::TMapType remapItems;
+ for (ui32 i = 1; i < input->ChildrenSize(); ++i) {
+ if (input->Child(i)->IsAtom()) {
+ ui32 index = FromString<ui32>(input->Child(i)->Content());
+ ++i;
+ if (outFromChildren.Test(i)) {
+ outConstraints.push_back(&input->Child(i)->GetConstraintSet());
+ if (const auto outMulti = input->Child(i)->GetConstraint<TMultiConstraintNode>()) {
+ for (auto& item: outMulti->GetItems()) {
+ remapItems.push_back(std::make_pair(item.first, index));
+ }
+ }
+ }
+ } else {
+ if (outFromChildren.Test(i)) {
+ outConstraints.push_back(&input->Child(i)->GetConstraintSet());
+ const auto outMulti = input->Child(i)->GetConstraint<TMultiConstraintNode>();
+ if (outMulti && inMulti) {
+ for (auto& outItem: outMulti->GetItems()) {
+ for (auto& inItem: inMulti->GetItems()) {
+ if (!usedAlts.Test(inItem.first)) {
+ remapItems.push_back(std::make_pair(outItem.first, inItem.first));
+ }
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (auto multi = TMultiConstraintNode::MakeCommon(outConstraints, ctx)) {
+ input->AddConstraint(multi);
+ }
+
+ if (auto empty = TEmptyConstraintNode::MakeCommon(outConstraints, ctx)) {
+ input->AddConstraint(empty);
+ }
+
+ if (auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>()) {
+ TVarIndexConstraintNode::TMapType varIndexItems;
+ for (auto& item: remapItems) {
+ const auto range = varIndex->GetIndexMapping().equal_range(item.second);
+ for (auto i = range.first; i != range.second; ++i) {
+ varIndexItems.push_back(std::make_pair(item.first, i->second));
+ }
+ }
+ if (!varIndexItems.empty()) {
+ ::Sort(varIndexItems);
+ input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(varIndexItems)));
+ }
+ }
+ }
+ else {
+ std::vector<const TConstraintSet*> nonEmpty;
+ for (ui32 i = 1; i < input->ChildrenSize(); ++i) {
+ if (input->Child(i)->IsAtom()) {
+ ++i;
+ }
+ if (outFromChildren.Test(i)) {
+ nonEmpty.push_back(&input->Child(i)->GetConstraintSet());
+ }
+ }
+ EraseIf(nonEmpty, [] (const TConstraintSet* c) { return !!c->GetConstraint<TEmptyConstraintNode>(); });
+
+ if (nonEmpty.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ } else if (nonEmpty.size() == 1) {
+ input->SetConstraints(*nonEmpty.front());
+ }
+
+ if (auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>()) {
+ TVarIndexConstraintNode::TMapType varIndexItems;
+ for (ui32 i = 1; i < input->ChildrenSize(); ++i) {
+ if (input->Child(i)->IsAtom()) {
const auto index = FromString<ui32>(input->Child(i++)->Content());
if (outFromChildren.Test(i) && IsDepended(input->Child(i)->Tail(), input->Child(i)->Head().Head())) { // Somehow depends on arg
const auto range = varIndex->GetIndexMapping().equal_range(index);
- for (auto i = range.first; i != range.second; ++i) {
- varIndexItems.push_back(std::make_pair(0, i->second));
- }
- }
- }
- else if (outFromChildren.Test(i)) {
- if (inMulti) {
- for (auto& inItem: inMulti->GetItems()) {
- if (!usedAlts.Test(inItem.first)) {
- auto range = varIndex->GetIndexMapping().equal_range(inItem.first);
- for (auto i = range.first; i != range.second; ++i) {
- varIndexItems.push_back(std::make_pair(0, i->second));
- }
- }
- }
- }
- }
- }
-
- if (!varIndexItems.empty()) {
- ::SortUnique(varIndexItems);
- input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(varIndexItems)));
- }
- }
-
- }
- return TStatus::Ok;
- }
-
- TStatus VariantItemWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- auto inputType = input->Child(0)->GetTypeAnn();
- if (inputType->GetKind() == ETypeAnnotationKind::Optional) {
- inputType = inputType->Cast<TOptionalExprType>()->GetItemType();
- }
-
- const auto underlyingType = inputType->Cast<TVariantExprType>()->GetUnderlyingType();
- if (underlyingType->GetKind() == ETypeAnnotationKind::Tuple) {
- if (auto multi = input->Child(0)->GetConstraint<TMultiConstraintNode>()) {
- std::vector<TMultiConstraintNode::TMapType::value_type> nonEmpty;
- std::copy_if(multi->GetItems().begin(), multi->GetItems().end(), std::back_inserter(nonEmpty),
- [] (const TMultiConstraintNode::TMapType::value_type& v) {
- return !v.second.GetConstraint<TEmptyConstraintNode>();
- }
- );
-
- if (nonEmpty.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- } else if (nonEmpty.size() == 1) {
- input->SetConstraints(nonEmpty.front().second);
- }
- }
- if (auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>()) {
- TVarIndexConstraintNode::TMapType varIndexItems;
- for (auto& item: varIndex->GetIndexMapping()) {
- varIndexItems.push_back(std::make_pair(0, item.second));
- }
- if (!varIndexItems.empty()) {
- ::SortUnique(varIndexItems);
- input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(varIndexItems)));
- }
- }
- }
- return TStatus::Ok;
- }
-
- TStatus VariantWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- if (input->GetTypeAnn()->Cast<TVariantExprType>()->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
- const auto index = FromString<ui32>(input->Child(1)->Content());
- TConstraintSet target;
- CopyExcept(target, input->Child(0)->GetConstraintSet(), TVarIndexConstraintNode::Name());
- input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(index, target));
- if (auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>()) {
- TVarIndexConstraintNode::TMapType filteredItems;
- for (auto& item: varIndex->GetIndexMapping()) {
- filteredItems.push_back(std::make_pair(index, item.second));
- }
- input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(filteredItems)));
- }
- }
- return TStatus::Ok;
- }
-
- TStatus GuessWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- auto inputType = input->Child(0)->GetTypeAnn();
- if (inputType->GetKind() == ETypeAnnotationKind::Optional) {
- inputType = inputType->Cast<TOptionalExprType>()->GetItemType();
- }
-
- const auto underlyingType = inputType->Cast<TVariantExprType>()->GetUnderlyingType();
- if (underlyingType->GetKind() == ETypeAnnotationKind::Tuple) {
- const auto guessIndex = FromString<ui32>(input->Child(1)->Content());
- if (auto multi = input->Child(0)->GetConstraint<TMultiConstraintNode>()) {
- if (auto c = multi->GetItem(guessIndex)) {
- input->SetConstraints(*c);
- } else {
- input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- }
- }
- if (auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>()) {
- TVarIndexConstraintNode::TMapType filteredItems;
- for (auto& item: varIndex->GetIndexMapping()) {
- if (item.first == guessIndex) {
- filteredItems.push_back(std::make_pair(0, item.second));
- }
- }
- if (!filteredItems.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(filteredItems)));
- }
- }
- }
- return TStatus::Ok;
- }
-
- TStatus MuxWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- if (input->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TVariantExprType>()->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
- if (input->Child(0)->IsList()) {
- TMultiConstraintNode::TMapType items;
- ui32 index = 0;
- for (auto& child: input->Child(0)->Children()) {
- items.push_back(std::make_pair(index, child->GetConstraintSet()));
- ++index;
- }
- if (!items.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(std::move(items)));
- }
- }
- }
- return TStatus::Ok;
- }
-
+ for (auto i = range.first; i != range.second; ++i) {
+ varIndexItems.push_back(std::make_pair(0, i->second));
+ }
+ }
+ }
+ else if (outFromChildren.Test(i)) {
+ if (inMulti) {
+ for (auto& inItem: inMulti->GetItems()) {
+ if (!usedAlts.Test(inItem.first)) {
+ auto range = varIndex->GetIndexMapping().equal_range(inItem.first);
+ for (auto i = range.first; i != range.second; ++i) {
+ varIndexItems.push_back(std::make_pair(0, i->second));
+ }
+ }
+ }
+ }
+ }
+ }
+
+ if (!varIndexItems.empty()) {
+ ::SortUnique(varIndexItems);
+ input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(varIndexItems)));
+ }
+ }
+
+ }
+ return TStatus::Ok;
+ }
+
+ TStatus VariantItemWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ auto inputType = input->Child(0)->GetTypeAnn();
+ if (inputType->GetKind() == ETypeAnnotationKind::Optional) {
+ inputType = inputType->Cast<TOptionalExprType>()->GetItemType();
+ }
+
+ const auto underlyingType = inputType->Cast<TVariantExprType>()->GetUnderlyingType();
+ if (underlyingType->GetKind() == ETypeAnnotationKind::Tuple) {
+ if (auto multi = input->Child(0)->GetConstraint<TMultiConstraintNode>()) {
+ std::vector<TMultiConstraintNode::TMapType::value_type> nonEmpty;
+ std::copy_if(multi->GetItems().begin(), multi->GetItems().end(), std::back_inserter(nonEmpty),
+ [] (const TMultiConstraintNode::TMapType::value_type& v) {
+ return !v.second.GetConstraint<TEmptyConstraintNode>();
+ }
+ );
+
+ if (nonEmpty.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ } else if (nonEmpty.size() == 1) {
+ input->SetConstraints(nonEmpty.front().second);
+ }
+ }
+ if (auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>()) {
+ TVarIndexConstraintNode::TMapType varIndexItems;
+ for (auto& item: varIndex->GetIndexMapping()) {
+ varIndexItems.push_back(std::make_pair(0, item.second));
+ }
+ if (!varIndexItems.empty()) {
+ ::SortUnique(varIndexItems);
+ input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(varIndexItems)));
+ }
+ }
+ }
+ return TStatus::Ok;
+ }
+
+ TStatus VariantWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ if (input->GetTypeAnn()->Cast<TVariantExprType>()->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
+ const auto index = FromString<ui32>(input->Child(1)->Content());
+ TConstraintSet target;
+ CopyExcept(target, input->Child(0)->GetConstraintSet(), TVarIndexConstraintNode::Name());
+ input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(index, target));
+ if (auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>()) {
+ TVarIndexConstraintNode::TMapType filteredItems;
+ for (auto& item: varIndex->GetIndexMapping()) {
+ filteredItems.push_back(std::make_pair(index, item.second));
+ }
+ input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(filteredItems)));
+ }
+ }
+ return TStatus::Ok;
+ }
+
+ TStatus GuessWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ auto inputType = input->Child(0)->GetTypeAnn();
+ if (inputType->GetKind() == ETypeAnnotationKind::Optional) {
+ inputType = inputType->Cast<TOptionalExprType>()->GetItemType();
+ }
+
+ const auto underlyingType = inputType->Cast<TVariantExprType>()->GetUnderlyingType();
+ if (underlyingType->GetKind() == ETypeAnnotationKind::Tuple) {
+ const auto guessIndex = FromString<ui32>(input->Child(1)->Content());
+ if (auto multi = input->Child(0)->GetConstraint<TMultiConstraintNode>()) {
+ if (auto c = multi->GetItem(guessIndex)) {
+ input->SetConstraints(*c);
+ } else {
+ input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ }
+ }
+ if (auto varIndex = input->Child(0)->GetConstraint<TVarIndexConstraintNode>()) {
+ TVarIndexConstraintNode::TMapType filteredItems;
+ for (auto& item: varIndex->GetIndexMapping()) {
+ if (item.first == guessIndex) {
+ filteredItems.push_back(std::make_pair(0, item.second));
+ }
+ }
+ if (!filteredItems.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(filteredItems)));
+ }
+ }
+ }
+ return TStatus::Ok;
+ }
+
+ TStatus MuxWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ if (input->GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TVariantExprType>()->GetUnderlyingType()->GetKind() == ETypeAnnotationKind::Tuple) {
+ if (input->Child(0)->IsList()) {
+ TMultiConstraintNode::TMapType items;
+ ui32 index = 0;
+ for (auto& child: input->Child(0)->Children()) {
+ items.push_back(std::make_pair(index, child->GetConstraintSet()));
+ ++index;
+ }
+ if (!items.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(std::move(items)));
+ }
+ }
+ }
+ return TStatus::Ok;
+ }
+
TStatus NthWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
TVector<TStringBuf> unique;
@@ -1920,119 +1920,119 @@ private:
input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(unique)));
}
- if (input->Child(0)->IsList()) {
- input->CopyConstraints(*input->Child(0)->Child(FromString<ui32>(input->Child(1)->Content())));
- }
- else if (input->Child(0)->IsCallable("Demux")) {
- if (auto multi = input->Child(0)->Child(0)->GetConstraint<TMultiConstraintNode>()) {
- if (auto c = multi->GetItem(FromString<ui32>(input->Child(1)->Content()))) {
- input->SetConstraints(*c);
- }
- }
- }
- return TStatus::Ok;
- }
-
- TStatus EquiJoinWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- const size_t numLists = input->ChildrenSize() - 2;
- TVector<size_t> emptyInputs;
- TJoinLabels labels;
- for (size_t i = 0; i < numLists; ++i) {
- auto& list = input->Child(i)->Head();
- if (list.GetConstraint<TEmptyConstraintNode>()) {
- emptyInputs.push_back(i);
- }
- if (auto err = labels.Add(ctx, *input->Child(i)->Child(1),
- list.GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>())) {
- ctx.AddError(*err);
- return TStatus::Error;
- }
- }
- if (emptyInputs.empty()) {
- return TStatus::Ok;
- }
-
- const auto joinTree = input->Child(numLists);
- for (auto i: emptyInputs) {
- if (IsRequiredSide(joinTree, labels, i).first) {
- input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- break;
- }
- }
-
- return TStatus::Ok;
- }
-
- TStatus MapJoinCoreWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& /*ctx*/) const {
- if (auto empty = input->Child(0)->GetConstraint<TEmptyConstraintNode>()) {
- input->AddConstraint(empty);
- } else if (auto empty = input->Child(1)->GetConstraint<TEmptyConstraintNode>()) {
- const auto joinType = input->Child(2)->Content();
- if (joinType == "Inner" || joinType == "LeftSemi") {
- input->AddConstraint(empty);
- }
- }
- return TStatus::Ok;
- }
-
- TStatus IsKeySwitchWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& /*ctx*/) const {
+ if (input->Child(0)->IsList()) {
+ input->CopyConstraints(*input->Child(0)->Child(FromString<ui32>(input->Child(1)->Content())));
+ }
+ else if (input->Child(0)->IsCallable("Demux")) {
+ if (auto multi = input->Child(0)->Child(0)->GetConstraint<TMultiConstraintNode>()) {
+ if (auto c = multi->GetItem(FromString<ui32>(input->Child(1)->Content()))) {
+ input->SetConstraints(*c);
+ }
+ }
+ }
+ return TStatus::Ok;
+ }
+
+ TStatus EquiJoinWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ const size_t numLists = input->ChildrenSize() - 2;
+ TVector<size_t> emptyInputs;
+ TJoinLabels labels;
+ for (size_t i = 0; i < numLists; ++i) {
+ auto& list = input->Child(i)->Head();
+ if (list.GetConstraint<TEmptyConstraintNode>()) {
+ emptyInputs.push_back(i);
+ }
+ if (auto err = labels.Add(ctx, *input->Child(i)->Child(1),
+ list.GetTypeAnn()->Cast<TListExprType>()->GetItemType()->Cast<TStructExprType>())) {
+ ctx.AddError(*err);
+ return TStatus::Error;
+ }
+ }
+ if (emptyInputs.empty()) {
+ return TStatus::Ok;
+ }
+
+ const auto joinTree = input->Child(numLists);
+ for (auto i: emptyInputs) {
+ if (IsRequiredSide(joinTree, labels, i).first) {
+ input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ break;
+ }
+ }
+
+ return TStatus::Ok;
+ }
+
+ TStatus MapJoinCoreWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& /*ctx*/) const {
+ if (auto empty = input->Child(0)->GetConstraint<TEmptyConstraintNode>()) {
+ input->AddConstraint(empty);
+ } else if (auto empty = input->Child(1)->GetConstraint<TEmptyConstraintNode>()) {
+ const auto joinType = input->Child(2)->Content();
+ if (joinType == "Inner" || joinType == "LeftSemi") {
+ input->AddConstraint(empty);
+ }
+ }
+ return TStatus::Ok;
+ }
+
+ TStatus IsKeySwitchWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& /*ctx*/) const {
if (const auto status = UpdateLambdaConstraints(*input->Child(TCoIsKeySwitch::idx_ItemKeyExtractor))
.Combine(UpdateLambdaConstraints(*input->Child(TCoIsKeySwitch::idx_StateKeyExtractor))); status != TStatus::Ok) {
- return status;
- }
-
+ return status;
+ }
+
if (const auto groupBy = input->Head().GetConstraint<TGroupByConstraintNode>()) {
- TVector<TStringBuf> keys;
- ExtractKeys(*input->Child(2), keys);
- if (!keys.empty()) {
- if (AllOf(keys, [groupBy] (TStringBuf key) { return groupBy->GetColumns().find(key) != groupBy->GetColumns().end(); })) {
- input->AddConstraint(groupBy);
- }
- }
- }
- return TStatus::Ok;
- }
-
- TStatus CondenseWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ TVector<TStringBuf> keys;
+ ExtractKeys(*input->Child(2), keys);
+ if (!keys.empty()) {
+ if (AllOf(keys, [groupBy] (TStringBuf key) { return groupBy->GetColumns().find(key) != groupBy->GetColumns().end(); })) {
+ input->AddConstraint(groupBy);
+ }
+ }
+ }
+ return TStatus::Ok;
+ }
+
+ TStatus CondenseWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
const TStructExprType* inItemType = GetNonEmptyStructItemType(*input->Head().GetTypeAnn());
- const TStructExprType* outItemType = GetNonEmptyStructItemType(*input->GetTypeAnn());
- if (!inItemType) {
- return InheriteEmptyFromInput(input, output, ctx);
- }
-
+ const TStructExprType* outItemType = GetNonEmptyStructItemType(*input->GetTypeAnn());
+ if (!inItemType) {
+ return InheriteEmptyFromInput(input, output, ctx);
+ }
+
const auto inputPassthrough = input->Head().GetConstraint<TPassthroughConstraintNode>();
- if (input->Child(1)->IsLambda()) {
- TConstraintNode::TListType argConstraints;
+ if (input->Child(1)->IsLambda()) {
+ TConstraintNode::TListType argConstraints;
if (inputPassthrough)
argConstraints.emplace_back(inputPassthrough);
if (const auto status = UpdateLambdaConstraints(input->ChildRef(1), ctx, {argConstraints}); status != TStatus::Ok) {
return status;
- }
- }
-
+ }
+ }
+
const auto initState = input->Child(1);
auto stateConstraints = initState->GetAllConstraints();
- stateConstraints.erase(
- std::remove_if(
- stateConstraints.begin(),
- stateConstraints.end(),
- [](const TConstraintNode* c) { return c->GetName() == TEmptyConstraintNode::Name(); }
- ),
- stateConstraints.end()
- );
-
- TConstraintNode::TListType itemConstraints;
+ stateConstraints.erase(
+ std::remove_if(
+ stateConstraints.begin(),
+ stateConstraints.end(),
+ [](const TConstraintNode* c) { return c->GetName() == TEmptyConstraintNode::Name(); }
+ ),
+ stateConstraints.end()
+ );
+
+ TConstraintNode::TListType itemConstraints;
if (inputPassthrough)
itemConstraints.emplace_back(inputPassthrough);
if (const auto groupBy = input->Head().GetConstraint<TGroupByConstraintNode>()) {
itemConstraints.push_back(groupBy);
- }
-
+ }
+
if (const auto status = UpdateLambdaConstraints(input->ChildRef(2), ctx, {itemConstraints, stateConstraints})
.Combine(UpdateLambdaConstraints(input->TailRef(), ctx, {itemConstraints, stateConstraints})); status != TStatus::Ok) {
- return status;
- }
-
+ return status;
+ }
+
const TPassthroughConstraintNode* commonPassthrough = nullptr;
const auto updateLambda = input->Child(3);
if (const auto lambdaPassthrough = updateLambda->GetConstraint<TPassthroughConstraintNode>()) {
@@ -2048,42 +2048,42 @@ private:
input->AddConstraint(commonPassthrough = lambdaPassthrough);
}
}
-
+
if (const auto switchLambda = input->Child(2); switchLambda->Tail().IsCallable(TCoBool::CallableName()) && IsFalse(switchLambda->Tail().Head().Content())) {
- if (outItemType) {
- input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(*outItemType));
- }
- }
- else {
- TVector<TStringBuf> groupByKeys;
+ if (outItemType) {
+ input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(*outItemType));
+ }
+ }
+ else {
+ TVector<TStringBuf> groupByKeys;
if (const auto groupBy = switchLambda->GetConstraint<TGroupByConstraintNode>()) {
- groupByKeys.assign(groupBy->GetColumns().begin(), groupBy->GetColumns().end());
+ groupByKeys.assign(groupBy->GetColumns().begin(), groupBy->GetColumns().end());
} else if (switchLambda->Tail().IsCallable({"AggrNotEquals", "NotEquals"})) {
- ExtractSimpleKeys(switchLambda->Child(1)->Child(0), switchLambda->Child(0)->Child(0), groupByKeys);
- }
+ ExtractSimpleKeys(switchLambda->Child(1)->Child(0), switchLambda->Child(0)->Child(0), groupByKeys);
+ }
if (!groupByKeys.empty() && commonPassthrough) {
const auto& mapping = commonPassthrough->GetReverseMapping();
- TUniqueConstraintNode::TSetType uniqColumns;
- for (auto key: groupByKeys) {
- auto range = mapping.equal_range(key);
- if (range.first != range.second) {
- for (auto i = range.first; i != range.second; ++i) {
- uniqColumns.insert_unique(i->second);
- }
- } else {
- uniqColumns.clear();
- break;
- }
- }
- if (!uniqColumns.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(uniqColumns)));
- }
- }
- }
-
- return FromFirst<TEmptyConstraintNode>(input, output, ctx);
- }
-
+ TUniqueConstraintNode::TSetType uniqColumns;
+ for (auto key: groupByKeys) {
+ auto range = mapping.equal_range(key);
+ if (range.first != range.second) {
+ for (auto i = range.first; i != range.second; ++i) {
+ uniqColumns.insert_unique(i->second);
+ }
+ } else {
+ uniqColumns.clear();
+ break;
+ }
+ }
+ if (!uniqColumns.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(uniqColumns)));
+ }
+ }
+ }
+
+ return FromFirst<TEmptyConstraintNode>(input, output, ctx);
+ }
+
TStatus WideCondense1Wrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
TSmallVec<TConstraintNode::TListType> argConstraints(input->Child(1)->Head().ChildrenSize());
const auto inputPassthrough = input->Head().GetConstraint<TPassthroughConstraintNode>();
@@ -2168,79 +2168,79 @@ private:
return FromFirst<TEmptyConstraintNode>(input, output, ctx);
}
- TStatus GroupByKeyWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ TStatus GroupByKeyWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
const TStructExprType* inItemType = GetNonEmptyStructItemType(*input->Head().GetTypeAnn());
- const TStructExprType* outItemType = GetNonEmptyStructItemType(*input->GetTypeAnn());
- if (inItemType && outItemType) {
+ const TStructExprType* outItemType = GetNonEmptyStructItemType(*input->GetTypeAnn());
+ if (inItemType && outItemType) {
const auto keySelector = input->Child(TCoGroupByKey::idx_KeySelectorLambda);
if (const auto status = UpdateLambdaConstraints(*keySelector); status != TStatus::Ok) {
return status;
- }
+ }
TConstraintNode::TListType argConstraints;
if (const auto inputPassthrough = input->Head().GetConstraint<TPassthroughConstraintNode>())
argConstraints.emplace_back(inputPassthrough);
if (const auto status = UpdateLambdaConstraints(input->ChildRef(TCoGroupByKey::idx_HandlerLambda), ctx, {TConstraintNode::TListType{}, argConstraints}); status != TStatus::Ok) {
return status;
}
-
+
if (const auto handlerLambda = input->Child(TCoGroupByKey::idx_HandlerLambda); handlerLambda->GetTypeAnn()->GetKind() == ETypeAnnotationKind::Optional) {
- TVector<TStringBuf> groupKeys;
- ExtractKeys(*keySelector, groupKeys);
- if (!groupKeys.empty()) {
+ TVector<TStringBuf> groupKeys;
+ ExtractKeys(*keySelector, groupKeys);
+ if (!groupKeys.empty()) {
if (const auto passthrough = handlerLambda->GetConstraint<TPassthroughConstraintNode>()) {
const auto mapping = passthrough->GetReverseMapping();
- TUniqueConstraintNode::TSetType uniqColumns;
- for (auto key: groupKeys) {
- auto range = mapping.equal_range(key);
- if (range.first != range.second) {
- for (auto i = range.first; i != range.second; ++i) {
- uniqColumns.insert_unique(i->second);
- }
- } else {
- uniqColumns.clear();
- break;
- }
- }
- if (!uniqColumns.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(uniqColumns)));
- }
- }
- }
- }
- } else {
- auto status = UpdateAllChildLambdasConstraints(*input);
- if (status != TStatus::Ok) {
- return status;
- }
- }
-
- TApplyConstraintFromInput<TCoGroupByKey::idx_HandlerLambda, TMultiConstraintNode, TEmptyConstraintNode>::Do(input);
- return FromFirst<TEmptyConstraintNode>(input, output, ctx);
- }
-
- TStatus PartitionByKeyWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
+ TUniqueConstraintNode::TSetType uniqColumns;
+ for (auto key: groupKeys) {
+ auto range = mapping.equal_range(key);
+ if (range.first != range.second) {
+ for (auto i = range.first; i != range.second; ++i) {
+ uniqColumns.insert_unique(i->second);
+ }
+ } else {
+ uniqColumns.clear();
+ break;
+ }
+ }
+ if (!uniqColumns.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(uniqColumns)));
+ }
+ }
+ }
+ }
+ } else {
+ auto status = UpdateAllChildLambdasConstraints(*input);
+ if (status != TStatus::Ok) {
+ return status;
+ }
+ }
+
+ TApplyConstraintFromInput<TCoGroupByKey::idx_HandlerLambda, TMultiConstraintNode, TEmptyConstraintNode>::Do(input);
+ return FromFirst<TEmptyConstraintNode>(input, output, ctx);
+ }
+
+ TStatus PartitionByKeyWrap(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) const {
if (const auto status = UpdateLambdaConstraints(*input->Child(TCoPartitionByKeyBase::idx_KeySelectorLambda)); status != TStatus::Ok) {
return status;
- }
+ }
if (const auto sortKeySelector = input->Child(TCoPartitionByKeyBase::idx_SortKeySelectorLambda); sortKeySelector->IsLambda()) {
if (const auto status = UpdateLambdaConstraints(*sortKeySelector); status != TStatus::Ok) {
return status;
}
}
-
+
std::unordered_set<const TPassthroughConstraintNode*> explicitPasstrought;
auto argConstraints = GetConstraintsForInputArgument(*input, explicitPasstrought, ctx);
-
- TVector<TStringBuf> partitionKeys;
- ExtractKeys(*input->Child(TCoPartitionByKeyBase::idx_KeySelectorLambda), partitionKeys);
- if (!partitionKeys.empty()) {
- argConstraints.push_back(ctx.MakeConstraint<TGroupByConstraintNode>(std::move(partitionKeys)));
- }
-
+
+ TVector<TStringBuf> partitionKeys;
+ ExtractKeys(*input->Child(TCoPartitionByKeyBase::idx_KeySelectorLambda), partitionKeys);
+ if (!partitionKeys.empty()) {
+ argConstraints.push_back(ctx.MakeConstraint<TGroupByConstraintNode>(std::move(partitionKeys)));
+ }
+
if (const auto status = UpdateLambdaConstraints(input->ChildRef(TCoPartitionByKeyBase::idx_ListHandlerLambda), ctx, {argConstraints}); status != TStatus::Ok) {
return status;
- }
-
+ }
+
const auto handlerLambda = input->Child(TCoPartitionByKeyBase::idx_ListHandlerLambda);
const auto lambdaPassthrough = handlerLambda->GetConstraint<TPassthroughConstraintNode>();
if (lambdaPassthrough) {
@@ -2254,63 +2254,63 @@ private:
}
}
- if (!partitionKeys.empty() && lambdaPassthrough) {
- if (auto uniq = handlerLambda->GetConstraint<TUniqueConstraintNode>()) {
- auto mapping = lambdaPassthrough->GetReverseMapping();
- TUniqueConstraintNode::TSetType uniqColumns;
- for (auto key: partitionKeys) {
- auto range = mapping.equal_range(key);
- if (range.first != range.second) {
- for (auto i = range.first; i != range.second; ++i) {
- uniqColumns.insert_unique(i->second);
- }
- } else {
- uniqColumns.clear();
- break;
- }
- }
- if (!uniqColumns.empty() && AllOf(uniqColumns, [uniq](TStringBuf key) { return uniq->GetColumns().has(key); })) {
- input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(uniqColumns)));
- }
- }
- }
-
+ if (!partitionKeys.empty() && lambdaPassthrough) {
+ if (auto uniq = handlerLambda->GetConstraint<TUniqueConstraintNode>()) {
+ auto mapping = lambdaPassthrough->GetReverseMapping();
+ TUniqueConstraintNode::TSetType uniqColumns;
+ for (auto key: partitionKeys) {
+ auto range = mapping.equal_range(key);
+ if (range.first != range.second) {
+ for (auto i = range.first; i != range.second; ++i) {
+ uniqColumns.insert_unique(i->second);
+ }
+ } else {
+ uniqColumns.clear();
+ break;
+ }
+ }
+ if (!uniqColumns.empty() && AllOf(uniqColumns, [uniq](TStringBuf key) { return uniq->GetColumns().has(key); })) {
+ input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(uniqColumns)));
+ }
+ }
+ }
+
const bool multiInput = ETypeAnnotationKind::Variant == GetItemType(*input->Head().GetTypeAnn())->GetKind();
- const auto lambdaVarIndex = handlerLambda->GetConstraint<TVarIndexConstraintNode>();
+ const auto lambdaVarIndex = handlerLambda->GetConstraint<TVarIndexConstraintNode>();
const auto multi = input->Head().GetConstraint<TMultiConstraintNode>();
- const auto lambdaMulti = handlerLambda->GetConstraint<TMultiConstraintNode>();
-
- if (const auto varIndex = input->Head().GetConstraint<TVarIndexConstraintNode>()) {
- if (multiInput) {
- if (lambdaVarIndex) {
- if (auto outVarIndex = GetVarIndexOverVarIndexConstraint(*varIndex, *lambdaVarIndex, ctx)) {
- input->AddConstraint(outVarIndex);
- }
- }
- } else {
- if (lambdaMulti) {
- TVarIndexConstraintNode::TMapType remapItems;
- for (auto& multiItem: lambdaMulti->GetItems()) {
- for (auto& varItem: varIndex->GetIndexMapping()) {
- remapItems.push_back(std::make_pair(multiItem.first, varItem.second));
- }
- }
- if (!remapItems.empty()) {
- ::SortUnique(remapItems);
- input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(remapItems)));
- }
- } else {
- input->AddConstraint(varIndex);
- }
- }
- }
-
- if (lambdaMulti && !input->Head().GetConstraint<TEmptyConstraintNode>()) {
- TMultiConstraintNode::TMapType remappedItems;
- for (auto& item: lambdaMulti->GetItems()) {
- remappedItems.push_back(std::make_pair(item.first, TConstraintSet{}));
- if (!multiInput) { // remapping one to many
- if (const auto lambdaPassthrough = item.second.template GetConstraint<TPassthroughConstraintNode>()) {
+ const auto lambdaMulti = handlerLambda->GetConstraint<TMultiConstraintNode>();
+
+ if (const auto varIndex = input->Head().GetConstraint<TVarIndexConstraintNode>()) {
+ if (multiInput) {
+ if (lambdaVarIndex) {
+ if (auto outVarIndex = GetVarIndexOverVarIndexConstraint(*varIndex, *lambdaVarIndex, ctx)) {
+ input->AddConstraint(outVarIndex);
+ }
+ }
+ } else {
+ if (lambdaMulti) {
+ TVarIndexConstraintNode::TMapType remapItems;
+ for (auto& multiItem: lambdaMulti->GetItems()) {
+ for (auto& varItem: varIndex->GetIndexMapping()) {
+ remapItems.push_back(std::make_pair(multiItem.first, varItem.second));
+ }
+ }
+ if (!remapItems.empty()) {
+ ::SortUnique(remapItems);
+ input->AddConstraint(ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(remapItems)));
+ }
+ } else {
+ input->AddConstraint(varIndex);
+ }
+ }
+ }
+
+ if (lambdaMulti && !input->Head().GetConstraint<TEmptyConstraintNode>()) {
+ TMultiConstraintNode::TMapType remappedItems;
+ for (auto& item: lambdaMulti->GetItems()) {
+ remappedItems.push_back(std::make_pair(item.first, TConstraintSet{}));
+ if (!multiInput) { // remapping one to many
+ if (const auto lambdaPassthrough = item.second.template GetConstraint<TPassthroughConstraintNode>()) {
if (!explicitPasstrought.contains(lambdaPassthrough)) {
auto mapping = lambdaPassthrough->GetColumnMapping();
for (const auto myPasstrought : explicitPasstrought)
@@ -2318,39 +2318,39 @@ private:
if (!mapping.empty()) {
remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(mapping)));
}
- }
- if (const auto lambdaUnique = item.second.template GetConstraint<TUniqueConstraintNode>()) {
- auto mapping = lambdaPassthrough->GetReverseMapping();
- TUniqueConstraintNode::TSetType uniqColumns;
- for (auto key: partitionKeys) {
- auto range = mapping.equal_range(key);
- if (range.first != range.second) {
- for (auto i = range.first; i != range.second; ++i) {
- uniqColumns.insert_unique(i->second);
- }
- } else {
- uniqColumns.clear();
- break;
- }
- }
- if (!uniqColumns.empty() && AllOf(uniqColumns, [lambdaUnique](TStringBuf key) { return lambdaUnique->GetColumns().has(key); })) {
- remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(uniqColumns)));
- }
- }
- }
-
- if (const auto empty = item.second.template GetConstraint<TEmptyConstraintNode>()) {
- remappedItems.pop_back();
- }
- }
- else if (lambdaVarIndex && multi) {
- const auto range = lambdaVarIndex->GetIndexMapping().equal_range(item.first);
- switch (std::distance(range.first, range.second)) {
- case 0: // new index
- break;
- case 1: // remapping 1 to 1
- if (auto origConstr = multi->GetItem(range.first->second)) {
- if (const auto lambdaPassthrough = item.second.template GetConstraint<TPassthroughConstraintNode>()) {
+ }
+ if (const auto lambdaUnique = item.second.template GetConstraint<TUniqueConstraintNode>()) {
+ auto mapping = lambdaPassthrough->GetReverseMapping();
+ TUniqueConstraintNode::TSetType uniqColumns;
+ for (auto key: partitionKeys) {
+ auto range = mapping.equal_range(key);
+ if (range.first != range.second) {
+ for (auto i = range.first; i != range.second; ++i) {
+ uniqColumns.insert_unique(i->second);
+ }
+ } else {
+ uniqColumns.clear();
+ break;
+ }
+ }
+ if (!uniqColumns.empty() && AllOf(uniqColumns, [lambdaUnique](TStringBuf key) { return lambdaUnique->GetColumns().has(key); })) {
+ remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(uniqColumns)));
+ }
+ }
+ }
+
+ if (const auto empty = item.second.template GetConstraint<TEmptyConstraintNode>()) {
+ remappedItems.pop_back();
+ }
+ }
+ else if (lambdaVarIndex && multi) {
+ const auto range = lambdaVarIndex->GetIndexMapping().equal_range(item.first);
+ switch (std::distance(range.first, range.second)) {
+ case 0: // new index
+ break;
+ case 1: // remapping 1 to 1
+ if (auto origConstr = multi->GetItem(range.first->second)) {
+ if (const auto lambdaPassthrough = item.second.template GetConstraint<TPassthroughConstraintNode>()) {
if (!explicitPasstrought.contains(lambdaPassthrough)) {
auto mapping = lambdaPassthrough->GetColumnMapping();
for (const auto myPasstrought : explicitPasstrought)
@@ -2358,177 +2358,177 @@ private:
if (!mapping.empty()) {
remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TPassthroughConstraintNode>(std::move(mapping)));
}
- }
- if (const auto lambdaUnique = item.second.template GetConstraint<TUniqueConstraintNode>()) {
- auto mapping = lambdaPassthrough->GetReverseMapping();
- TUniqueConstraintNode::TSetType uniqColumns;
- for (auto key: partitionKeys) {
- auto range = mapping.equal_range(key);
- if (range.first != range.second) {
- for (auto i = range.first; i != range.second; ++i) {
- uniqColumns.insert_unique(i->second);
- }
- } else {
- uniqColumns.clear();
- break;
- }
- }
- if (!uniqColumns.empty() && AllOf(uniqColumns, [lambdaUnique](TStringBuf key) { return lambdaUnique->GetColumns().has(key); })) {
- remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(uniqColumns)));
- }
- }
- }
- if (const auto empty = item.second.template GetConstraint<TEmptyConstraintNode>()) {
- remappedItems.pop_back();
- }
- } else {
- remappedItems.pop_back();
- }
- break;
- default: // remapping many to one
- {
- std::vector<const TConstraintSet*> nonEmpty;
- for (auto i = range.first; i != range.second; ++i) {
- if (auto origConstr = multi->GetItem(i->second)) {
- nonEmpty.push_back(origConstr);
- }
- }
- EraseIf(nonEmpty, [] (const TConstraintSet* c) { return !!c->GetConstraint<TEmptyConstraintNode>(); });
-
- if (nonEmpty.empty()) {
- remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- } else if (nonEmpty.size() == 1) {
- remappedItems.back().second = std::move(*nonEmpty.front());
- }
- }
- }
- } else {
- remappedItems.back().second = item.second;
- }
- }
- if (remappedItems) {
- input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(std::move(remappedItems)));
- }
- }
- else if (multi && lambdaVarIndex) { // Many to one
- const auto range = lambdaVarIndex->GetIndexMapping().equal_range(0);
- std::vector<const TConstraintSet*> nonEmpty;
- for (auto i = range.first; i != range.second; ++i) {
- if (auto origConstr = multi->GetItem(i->second)) {
- nonEmpty.push_back(origConstr);
- }
- }
- EraseIf(nonEmpty, [] (const TConstraintSet* c) { return !!c->GetConstraint<TEmptyConstraintNode>(); });
-
- if (nonEmpty.empty()) {
- input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
- } else if (nonEmpty.size() == 1) {
- input->SetConstraints(*nonEmpty.front());
- }
- }
-
- TApplyConstraintFromInput<TCoPartitionByKeyBase::idx_ListHandlerLambda, TEmptyConstraintNode>::Do(input);
- return FromFirst<TEmptyConstraintNode>(input, output, ctx);
- }
-
- TStatus AggregateWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
- if (input->Child(1)->ChildrenSize()) {
- TUniqueConstraintNode::TSetType columns;
- for (auto& child: input->Child(1)->Children()) {
- columns.insert(child->Content());
- }
- input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(columns)));
- }
- return TStatus::Ok;
- }
-
+ }
+ if (const auto lambdaUnique = item.second.template GetConstraint<TUniqueConstraintNode>()) {
+ auto mapping = lambdaPassthrough->GetReverseMapping();
+ TUniqueConstraintNode::TSetType uniqColumns;
+ for (auto key: partitionKeys) {
+ auto range = mapping.equal_range(key);
+ if (range.first != range.second) {
+ for (auto i = range.first; i != range.second; ++i) {
+ uniqColumns.insert_unique(i->second);
+ }
+ } else {
+ uniqColumns.clear();
+ break;
+ }
+ }
+ if (!uniqColumns.empty() && AllOf(uniqColumns, [lambdaUnique](TStringBuf key) { return lambdaUnique->GetColumns().has(key); })) {
+ remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(uniqColumns)));
+ }
+ }
+ }
+ if (const auto empty = item.second.template GetConstraint<TEmptyConstraintNode>()) {
+ remappedItems.pop_back();
+ }
+ } else {
+ remappedItems.pop_back();
+ }
+ break;
+ default: // remapping many to one
+ {
+ std::vector<const TConstraintSet*> nonEmpty;
+ for (auto i = range.first; i != range.second; ++i) {
+ if (auto origConstr = multi->GetItem(i->second)) {
+ nonEmpty.push_back(origConstr);
+ }
+ }
+ EraseIf(nonEmpty, [] (const TConstraintSet* c) { return !!c->GetConstraint<TEmptyConstraintNode>(); });
+
+ if (nonEmpty.empty()) {
+ remappedItems.back().second.AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ } else if (nonEmpty.size() == 1) {
+ remappedItems.back().second = std::move(*nonEmpty.front());
+ }
+ }
+ }
+ } else {
+ remappedItems.back().second = item.second;
+ }
+ }
+ if (remappedItems) {
+ input->AddConstraint(ctx.MakeConstraint<TMultiConstraintNode>(std::move(remappedItems)));
+ }
+ }
+ else if (multi && lambdaVarIndex) { // Many to one
+ const auto range = lambdaVarIndex->GetIndexMapping().equal_range(0);
+ std::vector<const TConstraintSet*> nonEmpty;
+ for (auto i = range.first; i != range.second; ++i) {
+ if (auto origConstr = multi->GetItem(i->second)) {
+ nonEmpty.push_back(origConstr);
+ }
+ }
+ EraseIf(nonEmpty, [] (const TConstraintSet* c) { return !!c->GetConstraint<TEmptyConstraintNode>(); });
+
+ if (nonEmpty.empty()) {
+ input->AddConstraint(ctx.MakeConstraint<TEmptyConstraintNode>());
+ } else if (nonEmpty.size() == 1) {
+ input->SetConstraints(*nonEmpty.front());
+ }
+ }
+
+ TApplyConstraintFromInput<TCoPartitionByKeyBase::idx_ListHandlerLambda, TEmptyConstraintNode>::Do(input);
+ return FromFirst<TEmptyConstraintNode>(input, output, ctx);
+ }
+
+ TStatus AggregateWrap(const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) const {
+ if (input->Child(1)->ChildrenSize()) {
+ TUniqueConstraintNode::TSetType columns;
+ for (auto& child: input->Child(1)->Children()) {
+ columns.insert(child->Content());
+ }
+ input->AddConstraint(ctx.MakeConstraint<TUniqueConstraintNode>(std::move(columns)));
+ }
+ return TStatus::Ok;
+ }
+
TStatus FoldWrap(const TExprNode::TPtr& input, TExprNode::TPtr&, TExprContext& ctx) const {
- const TStructExprType* inItemType = GetNonEmptyStructItemType(*input->Child(0)->GetTypeAnn());
- const TStructExprType* outItemType = GetNonEmptyStructItemType(*input->GetTypeAnn());
- if (!inItemType || !outItemType) {
- return UpdateAllChildLambdasConstraints(*input);
- }
-
+ const TStructExprType* inItemType = GetNonEmptyStructItemType(*input->Child(0)->GetTypeAnn());
+ const TStructExprType* outItemType = GetNonEmptyStructItemType(*input->GetTypeAnn());
+ if (!inItemType || !outItemType) {
+ return UpdateAllChildLambdasConstraints(*input);
+ }
+
const auto inputPassthrough = input->Head().GetConstraint<TPassthroughConstraintNode>();
- if (input->Child(1)->IsLambda()) {
+ if (input->Child(1)->IsLambda()) {
TConstraintNode::TListType argConstraints;
if (inputPassthrough)
argConstraints.emplace_back(inputPassthrough);
if (const auto status = UpdateLambdaConstraints(input->ChildRef(1), ctx, {argConstraints}); status != TStatus::Ok) {
return status;
- }
- }
-
+ }
+ }
+
const auto initState = input->Child(1);
auto stateConstraints = initState->GetAllConstraints();
- stateConstraints.erase(
- std::remove_if(
- stateConstraints.begin(),
- stateConstraints.end(),
- [](const TConstraintNode* c) { return c->GetName() == TEmptyConstraintNode::Name(); }
- ),
- stateConstraints.end()
- );
-
+ stateConstraints.erase(
+ std::remove_if(
+ stateConstraints.begin(),
+ stateConstraints.end(),
+ [](const TConstraintNode* c) { return c->GetName() == TEmptyConstraintNode::Name(); }
+ ),
+ stateConstraints.end()
+ );
+
TConstraintNode::TListType argConstraints;
if (inputPassthrough)
argConstraints.emplace_back(inputPassthrough);
-
+
if (const auto status = UpdateLambdaConstraints(input->TailRef(), ctx, {argConstraints, stateConstraints}); status != TStatus::Ok) {
- return status;
- }
-
+ return status;
+ }
+
if (const auto lambdaPassthrough = input->Tail().GetConstraint<TPassthroughConstraintNode>()) {
input->AddConstraint(lambdaPassthrough);
- }
-
- return TStatus::Ok;
- }
-private:
- template <class TConstraintContainer>
- static void CopyExcept(TConstraintContainer& dst, const TConstraintContainer& from, const TSet<TStringBuf>& except) {
- for (auto c: from.GetAllConstraints()) {
- if (!except.contains(c->GetName())) {
- dst.AddConstraint(c);
- }
- }
- }
-
- template <class TConstraintContainer>
- static void CopyExcept(TConstraintContainer& dst, const TConstraintContainer& from, TStringBuf except) {
- for (auto c: from.GetAllConstraints()) {
- if (c->GetName() != except) {
- dst.AddConstraint(c);
- }
- }
- }
-
- static void ExtractKeys(const TExprNode& keySelectorLambda, TVector<TStringBuf>& columns) {
- auto arg = keySelectorLambda.Child(0)->Child(0);
- auto body = keySelectorLambda.Child(1);
- if (body->IsCallable("StablePickle")) {
- body = body->Child(0);
- }
- ExtractSimpleKeys(body, arg, columns);
- }
-
- static const TStructExprType* GetNonEmptyStructItemType(const TTypeAnnotationNode& type) {
- const TTypeAnnotationNode* itemType = GetItemType(type);
- if (!itemType || itemType->GetKind() != ETypeAnnotationKind::Struct) {
- return nullptr;
- }
- const TStructExprType* structType = itemType->Cast<TStructExprType>();
- return structType->GetSize() ? structType : nullptr;
- }
-
- static const TSortedConstraintNode* DeduceSortConstraint(const TExprNode& list, const TExprNode& directions, const TExprNode& keyExtractor, TExprContext& ctx) {
- // Ignore Sort with empty key tuple
- if (directions.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple &&
- directions.GetTypeAnn()->Cast<TTupleExprType>()->GetSize() == 0) {
-
- return nullptr;
- }
-
+ }
+
+ return TStatus::Ok;
+ }
+private:
+ template <class TConstraintContainer>
+ static void CopyExcept(TConstraintContainer& dst, const TConstraintContainer& from, const TSet<TStringBuf>& except) {
+ for (auto c: from.GetAllConstraints()) {
+ if (!except.contains(c->GetName())) {
+ dst.AddConstraint(c);
+ }
+ }
+ }
+
+ template <class TConstraintContainer>
+ static void CopyExcept(TConstraintContainer& dst, const TConstraintContainer& from, TStringBuf except) {
+ for (auto c: from.GetAllConstraints()) {
+ if (c->GetName() != except) {
+ dst.AddConstraint(c);
+ }
+ }
+ }
+
+ static void ExtractKeys(const TExprNode& keySelectorLambda, TVector<TStringBuf>& columns) {
+ auto arg = keySelectorLambda.Child(0)->Child(0);
+ auto body = keySelectorLambda.Child(1);
+ if (body->IsCallable("StablePickle")) {
+ body = body->Child(0);
+ }
+ ExtractSimpleKeys(body, arg, columns);
+ }
+
+ static const TStructExprType* GetNonEmptyStructItemType(const TTypeAnnotationNode& type) {
+ const TTypeAnnotationNode* itemType = GetItemType(type);
+ if (!itemType || itemType->GetKind() != ETypeAnnotationKind::Struct) {
+ return nullptr;
+ }
+ const TStructExprType* structType = itemType->Cast<TStructExprType>();
+ return structType->GetSize() ? structType : nullptr;
+ }
+
+ static const TSortedConstraintNode* DeduceSortConstraint(const TExprNode& list, const TExprNode& directions, const TExprNode& keyExtractor, TExprContext& ctx) {
+ // Ignore Sort with empty key tuple
+ if (directions.GetTypeAnn()->GetKind() == ETypeAnnotationKind::Tuple &&
+ directions.GetTypeAnn()->Cast<TTupleExprType>()->GetSize() == 0) {
+
+ return nullptr;
+ }
+
if (GetSeqItemType(list.GetTypeAnn())->GetKind() == ETypeAnnotationKind::Struct) {
TVector<bool> dirs;
TVector<TStringBuf> keys;
@@ -2539,30 +2539,30 @@ private:
content.emplace_back(TSortedConstraintNode::TContainerType::value_type::first_type{keys[i]}, dirs[i]);
if (!content.empty()) {
return ctx.MakeConstraint<TSortedConstraintNode>(std::move(content));
- }
- }
- return nullptr;
- }
-
- static const TVarIndexConstraintNode* GetVarIndexOverVarIndexConstraint(const TVarIndexConstraintNode& inputVarIndex,
- const TVarIndexConstraintNode& varIndex, TExprContext& ctx)
- {
- TVarIndexConstraintNode::TMapType result;
- for (auto& item: varIndex.GetIndexMapping()) {
- auto range = inputVarIndex.GetIndexMapping().equal_range(item.second);
- for (auto it = range.first; it != range.second; ++it) {
- result.push_back(std::make_pair(item.first, it->second));
- }
- }
- if (!result.empty()) {
- return ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(result));
- }
- return nullptr;
- }
-
- static const TSortedConstraintNode* GetPassthroughSortedConstraint(const TSortedConstraintNode& inputSorted,
- const TPassthroughConstraintNode& passthrough, TExprContext& ctx)
- {
+ }
+ }
+ return nullptr;
+ }
+
+ static const TVarIndexConstraintNode* GetVarIndexOverVarIndexConstraint(const TVarIndexConstraintNode& inputVarIndex,
+ const TVarIndexConstraintNode& varIndex, TExprContext& ctx)
+ {
+ TVarIndexConstraintNode::TMapType result;
+ for (auto& item: varIndex.GetIndexMapping()) {
+ auto range = inputVarIndex.GetIndexMapping().equal_range(item.second);
+ for (auto it = range.first; it != range.second; ++it) {
+ result.push_back(std::make_pair(item.first, it->second));
+ }
+ }
+ if (!result.empty()) {
+ return ctx.MakeConstraint<TVarIndexConstraintNode>(std::move(result));
+ }
+ return nullptr;
+ }
+
+ static const TSortedConstraintNode* GetPassthroughSortedConstraint(const TSortedConstraintNode& inputSorted,
+ const TPassthroughConstraintNode& passthrough, TExprContext& ctx)
+ {
const auto& reverseMapping = passthrough.GetReverseMapping();
const auto& content = inputSorted.GetContent();
TSortedConstraintNode::TContainerType filtered;
@@ -2574,59 +2574,59 @@ private:
for (auto it = range.first; it != range.second; ++it) {
nextItem.first.emplace_back(it->second);
}
- }
+ }
}
if (nextItem.first.empty())
- break;
+ break;
nextItem.second = content[i].second;
::Sort(nextItem.first);
filtered.emplace_back(std::move(nextItem));
- }
+ }
if (!filtered.empty()) {
return ctx.MakeConstraint<TSortedConstraintNode>(std::move(filtered));
- }
- return nullptr;
- }
-
- static const TUniqueConstraintNode* GetPassthroughUniqueConstraint(const TUniqueConstraintNode& inputUnique,
- const TPassthroughConstraintNode& passthrough, TExprContext& ctx)
- {
+ }
+ return nullptr;
+ }
+
+ static const TUniqueConstraintNode* GetPassthroughUniqueConstraint(const TUniqueConstraintNode& inputUnique,
+ const TPassthroughConstraintNode& passthrough, TExprContext& ctx)
+ {
const auto reverseMapping = passthrough.GetReverseMapping();
- TUniqueConstraintNode::TSetType remappedColumns;
- for (auto col: inputUnique.GetColumns()) {
- auto range = reverseMapping.equal_range(col);
- if (range.first != range.second) {
- for (auto it = range.first; it != range.second; ++it) {
- remappedColumns.insert(it->second);
- }
- } else {
- remappedColumns.clear();
- break;
- }
- }
- if (remappedColumns) {
- return ctx.MakeConstraint<TUniqueConstraintNode>(std::move(remappedColumns));
- }
- return nullptr;
- }
-
- static const TExprNode* SkipModifiers(const TExprNode* valueNode) {
- if (TCoJust::Match(valueNode)) {
- return SkipModifiers(valueNode->Child(0));
- }
- if (TCoUnwrap::Match(valueNode)) {
- return SkipModifiers(valueNode->Child(0));
- }
- return valueNode;
- }
-
-private:
- const bool SubGraph;
- THashMap<TStringBuf, THandler> Functions;
-};
-
+ TUniqueConstraintNode::TSetType remappedColumns;
+ for (auto col: inputUnique.GetColumns()) {
+ auto range = reverseMapping.equal_range(col);
+ if (range.first != range.second) {
+ for (auto it = range.first; it != range.second; ++it) {
+ remappedColumns.insert(it->second);
+ }
+ } else {
+ remappedColumns.clear();
+ break;
+ }
+ }
+ if (remappedColumns) {
+ return ctx.MakeConstraint<TUniqueConstraintNode>(std::move(remappedColumns));
+ }
+ return nullptr;
+ }
+
+ static const TExprNode* SkipModifiers(const TExprNode* valueNode) {
+ if (TCoJust::Match(valueNode)) {
+ return SkipModifiers(valueNode->Child(0));
+ }
+ if (TCoUnwrap::Match(valueNode)) {
+ return SkipModifiers(valueNode->Child(0));
+ }
+ return valueNode;
+ }
+
+private:
+ const bool SubGraph;
+ THashMap<TStringBuf, THandler> Functions;
+};
+
template<> const TPassthroughConstraintNode*
TCallableConstraintTransformer::GetConstraintFromWideResultLambda<TPassthroughConstraintNode>(const TExprNode& lambda, TExprContext& ctx) {
TPassthroughConstraintNode::TMapType passthrough;
@@ -2723,217 +2723,217 @@ TCallableConstraintTransformer::TCallableConstraintTransformer::GetConstraintFro
template<class TConstraint> const TConstraint*
TCallableConstraintTransformer::GetConstraintFromWideResultLambda(const TExprNode&, TExprContext&) { return nullptr; }
-class TDefaultCallableConstraintTransformer : public TSyncTransformerBase {
-public:
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override {
- Y_UNUSED(output);
- Y_UNUSED(ctx);
- return UpdateAllChildLambdasConstraints(*input);
- }
-};
-
-class TConstraintTransformer : public TGraphTransformerBase {
-public:
- TConstraintTransformer(TAutoPtr<IGraphTransformer> callableTransformer, TTypeAnnotationContext& types)
- : CallableTransformer(callableTransformer)
- , Types(types)
- {
- }
-
- ~TConstraintTransformer() = default;
-
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- YQL_PROFILE_SCOPE(DEBUG, "ConstraintTransformer::DoTransform");
- output = input;
- auto status = TransformNode(input, output, ctx);
- UpdateStatusIfChanged(status, input, output);
-
- if (status.Level != TStatus::Error && HasRenames) {
+class TDefaultCallableConstraintTransformer : public TSyncTransformerBase {
+public:
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override {
+ Y_UNUSED(output);
+ Y_UNUSED(ctx);
+ return UpdateAllChildLambdasConstraints(*input);
+ }
+};
+
+class TConstraintTransformer : public TGraphTransformerBase {
+public:
+ TConstraintTransformer(TAutoPtr<IGraphTransformer> callableTransformer, TTypeAnnotationContext& types)
+ : CallableTransformer(callableTransformer)
+ , Types(types)
+ {
+ }
+
+ ~TConstraintTransformer() = default;
+
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ YQL_PROFILE_SCOPE(DEBUG, "ConstraintTransformer::DoTransform");
+ output = input;
+ auto status = TransformNode(input, output, ctx);
+ UpdateStatusIfChanged(status, input, output);
+
+ if (status.Level != TStatus::Error && HasRenames) {
output = ctx.ReplaceNodes(std::move(output), Processed);
- }
-
- Processed.clear();
- if (status == TStatus::Ok) {
- Types.ExpectedConstraints.clear();
- }
-
- HasRenames = false;
- return status;
- }
-
- NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final {
- YQL_PROFILE_SCOPE(DEBUG, "ConstraintTransformer::DoGetAsyncFuture");
- Y_UNUSED(input);
- TVector<NThreading::TFuture<void>> futures;
- for (const auto& callable : CallableInputs) {
- futures.push_back(CallableTransformer->GetAsyncFuture(*callable));
- }
-
+ }
+
+ Processed.clear();
+ if (status == TStatus::Ok) {
+ Types.ExpectedConstraints.clear();
+ }
+
+ HasRenames = false;
+ return status;
+ }
+
+ NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final {
+ YQL_PROFILE_SCOPE(DEBUG, "ConstraintTransformer::DoGetAsyncFuture");
+ Y_UNUSED(input);
+ TVector<NThreading::TFuture<void>> futures;
+ for (const auto& callable : CallableInputs) {
+ futures.push_back(CallableTransformer->GetAsyncFuture(*callable));
+ }
+
return WaitExceptionOrAll(futures);
- }
-
- TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- YQL_PROFILE_SCOPE(DEBUG, "ConstraintTransformer::DoApplyAsyncChanges");
- output = input;
- TStatus combinedStatus = TStatus::Ok;
- for (const auto& callable : CallableInputs) {
+ }
+
+ TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ YQL_PROFILE_SCOPE(DEBUG, "ConstraintTransformer::DoApplyAsyncChanges");
+ output = input;
+ TStatus combinedStatus = TStatus::Ok;
+ for (const auto& callable : CallableInputs) {
callable->SetState(TExprNode::EState::ConstrPending);
- TExprNode::TPtr callableOutput;
- auto status = CallableTransformer->ApplyAsyncChanges(callable, callableOutput, ctx);
- Y_VERIFY(callableOutput);
- YQL_ENSURE(status != TStatus::Async);
- YQL_ENSURE(callableOutput == callable);
- combinedStatus = combinedStatus.Combine(status);
- if (status.Level == TStatus::Error) {
+ TExprNode::TPtr callableOutput;
+ auto status = CallableTransformer->ApplyAsyncChanges(callable, callableOutput, ctx);
+ Y_VERIFY(callableOutput);
+ YQL_ENSURE(status != TStatus::Async);
+ YQL_ENSURE(callableOutput == callable);
+ combinedStatus = combinedStatus.Combine(status);
+ if (status.Level == TStatus::Error) {
callable->SetState(TExprNode::EState::Error);
- }
- }
-
- CallableInputs.clear();
- if (combinedStatus.Level == TStatus::Ok) {
- Processed.clear();
- }
-
- return combinedStatus;
- }
-
-private:
- TStatus TransformNode(const TExprNode::TPtr& start, TExprNode::TPtr& output, TExprContext& ctx) {
- output = start;
- auto processedPair = Processed.emplace(start.Get(), nullptr); // by default node is not changed
- if (!processedPair.second) {
- if (processedPair.first->second) {
- output = processedPair.first->second;
- return TStatus::Repeat;
- }
-
+ }
+ }
+
+ CallableInputs.clear();
+ if (combinedStatus.Level == TStatus::Ok) {
+ Processed.clear();
+ }
+
+ return combinedStatus;
+ }
+
+private:
+ TStatus TransformNode(const TExprNode::TPtr& start, TExprNode::TPtr& output, TExprContext& ctx) {
+ output = start;
+ auto processedPair = Processed.emplace(start.Get(), nullptr); // by default node is not changed
+ if (!processedPair.second) {
+ if (processedPair.first->second) {
+ output = processedPair.first->second;
+ return TStatus::Repeat;
+ }
+
switch (start->GetState()) {
- case TExprNode::EState::Initial:
- case TExprNode::EState::TypeInProgress:
- case TExprNode::EState::TypePending:
- return TStatus(TStatus::Repeat, true);
- case TExprNode::EState::TypeComplete:
- break;
- case TExprNode::EState::ConstrInProgress:
- return IGraphTransformer::TStatus::Async;
- case TExprNode::EState::ConstrPending:
- if (start->Type() == TExprNode::Lambda) {
+ case TExprNode::EState::Initial:
+ case TExprNode::EState::TypeInProgress:
+ case TExprNode::EState::TypePending:
+ return TStatus(TStatus::Repeat, true);
+ case TExprNode::EState::TypeComplete:
+ break;
+ case TExprNode::EState::ConstrInProgress:
+ return IGraphTransformer::TStatus::Async;
+ case TExprNode::EState::ConstrPending:
+ if (start->Type() == TExprNode::Lambda) {
if (start->Child(0)->GetState() != TExprNode::EState::ConstrComplete) {
- return TStatus::Ok;
- } else if (start->Child(0)->ChildrenSize() == 0) {
- break;
- }
- }
-
- if (start->Type() == TExprNode::Arguments || start->Type() == TExprNode::Argument) {
- break;
- }
-
- return TStatus(TStatus::Repeat, true);
- case TExprNode::EState::ConstrComplete:
- case TExprNode::EState::ExecutionInProgress:
- case TExprNode::EState::ExecutionRequired:
- case TExprNode::EState::ExecutionPending:
- case TExprNode::EState::ExecutionComplete:
- return TStatus::Ok;
- case TExprNode::EState::Error:
- return TStatus::Error;
- default:
- YQL_ENSURE(false, "Unknown state");
- }
- }
-
- auto input = start;
- for (;;) {
+ return TStatus::Ok;
+ } else if (start->Child(0)->ChildrenSize() == 0) {
+ break;
+ }
+ }
+
+ if (start->Type() == TExprNode::Arguments || start->Type() == TExprNode::Argument) {
+ break;
+ }
+
+ return TStatus(TStatus::Repeat, true);
+ case TExprNode::EState::ConstrComplete:
+ case TExprNode::EState::ExecutionInProgress:
+ case TExprNode::EState::ExecutionRequired:
+ case TExprNode::EState::ExecutionPending:
+ case TExprNode::EState::ExecutionComplete:
+ return TStatus::Ok;
+ case TExprNode::EState::Error:
+ return TStatus::Error;
+ default:
+ YQL_ENSURE(false, "Unknown state");
+ }
+ }
+
+ auto input = start;
+ for (;;) {
TIssueScopeGuard issueScope(ctx.IssueManager, [this, input, &ctx]() -> TIssuePtr {
- TStringBuilder str;
- str << "At ";
- switch (input->Type()) {
- case TExprNode::Callable:
- if (!CurrentFunctions.empty() && CurrentFunctions.top().second) {
- return nullptr;
- }
-
- if (!CurrentFunctions.empty()) {
- CurrentFunctions.top().second = true;
- }
-
- str << "function: " << input->Content();
- break;
- case TExprNode::List:
- if (CurrentFunctions.empty()) {
- str << "tuple";
- } else if (!CurrentFunctions.top().second) {
- CurrentFunctions.top().second = true;
- str << "function: " << CurrentFunctions.top().first;
- } else {
- return nullptr;
- }
- break;
- case TExprNode::Lambda:
- if (CurrentFunctions.empty()) {
- str << "lambda";
- } else if (!CurrentFunctions.top().second) {
- CurrentFunctions.top().second = true;
- str << "function: " << CurrentFunctions.top().first;
- } else {
- return nullptr;
- }
- break;
- default:
- str << "unknown";
- }
-
+ TStringBuilder str;
+ str << "At ";
+ switch (input->Type()) {
+ case TExprNode::Callable:
+ if (!CurrentFunctions.empty() && CurrentFunctions.top().second) {
+ return nullptr;
+ }
+
+ if (!CurrentFunctions.empty()) {
+ CurrentFunctions.top().second = true;
+ }
+
+ str << "function: " << input->Content();
+ break;
+ case TExprNode::List:
+ if (CurrentFunctions.empty()) {
+ str << "tuple";
+ } else if (!CurrentFunctions.top().second) {
+ CurrentFunctions.top().second = true;
+ str << "function: " << CurrentFunctions.top().first;
+ } else {
+ return nullptr;
+ }
+ break;
+ case TExprNode::Lambda:
+ if (CurrentFunctions.empty()) {
+ str << "lambda";
+ } else if (!CurrentFunctions.top().second) {
+ CurrentFunctions.top().second = true;
+ str << "function: " << CurrentFunctions.top().first;
+ } else {
+ return nullptr;
+ }
+ break;
+ default:
+ str << "unknown";
+ }
+
return MakeIntrusive<TIssue>(ctx.GetPosition(input->Pos()), str);
- });
-
- if (input->Type() == TExprNode::Callable) {
- CurrentFunctions.push(std::make_pair(input->Content(), false));
- }
- Y_SCOPE_EXIT(this, input) {
- if (input->Type() == TExprNode::Callable) {
- CurrentFunctions.pop();
- if (!CurrentFunctions.empty() && CurrentFunctions.top().first.EndsWith('!')) {
- CurrentFunctions.top().second = true;
- }
- }
- };
-
- TStatus retStatus = TStatus::Error;
+ });
+
+ if (input->Type() == TExprNode::Callable) {
+ CurrentFunctions.push(std::make_pair(input->Content(), false));
+ }
+ Y_SCOPE_EXIT(this, input) {
+ if (input->Type() == TExprNode::Callable) {
+ CurrentFunctions.pop();
+ if (!CurrentFunctions.empty() && CurrentFunctions.top().first.EndsWith('!')) {
+ CurrentFunctions.top().second = true;
+ }
+ }
+ };
+
+ TStatus retStatus = TStatus::Error;
switch (input->GetState()) {
- case TExprNode::EState::Initial:
- case TExprNode::EState::TypeInProgress:
- case TExprNode::EState::TypePending:
- return TStatus(TStatus::Repeat, true);
- case TExprNode::EState::TypeComplete:
- case TExprNode::EState::ConstrPending:
- break;
- case TExprNode::EState::ConstrInProgress:
- return IGraphTransformer::TStatus::Async;
- case TExprNode::EState::ConstrComplete:
- case TExprNode::EState::ExecutionInProgress:
- case TExprNode::EState::ExecutionRequired:
- case TExprNode::EState::ExecutionPending:
- case TExprNode::EState::ExecutionComplete:
- return TStatus::Ok;
- case TExprNode::EState::Error:
- return TStatus::Error;
- default:
- YQL_ENSURE(false, "Unknown state");
- }
-
+ case TExprNode::EState::Initial:
+ case TExprNode::EState::TypeInProgress:
+ case TExprNode::EState::TypePending:
+ return TStatus(TStatus::Repeat, true);
+ case TExprNode::EState::TypeComplete:
+ case TExprNode::EState::ConstrPending:
+ break;
+ case TExprNode::EState::ConstrInProgress:
+ return IGraphTransformer::TStatus::Async;
+ case TExprNode::EState::ConstrComplete:
+ case TExprNode::EState::ExecutionInProgress:
+ case TExprNode::EState::ExecutionRequired:
+ case TExprNode::EState::ExecutionPending:
+ case TExprNode::EState::ExecutionComplete:
+ return TStatus::Ok;
+ case TExprNode::EState::Error:
+ return TStatus::Error;
+ default:
+ YQL_ENSURE(false, "Unknown state");
+ }
+
input->SetState(TExprNode::EState::ConstrPending);
- switch (input->Type()) {
- case TExprNode::Atom:
- case TExprNode::World:
+ switch (input->Type()) {
+ case TExprNode::Atom:
+ case TExprNode::World:
input->SetState(TExprNode::EState::ConstrComplete);
- CheckExpected(*input);
- return TStatus::Ok;
-
- case TExprNode::List:
- {
- retStatus = TransformChildren(input, output, ctx);
- if (retStatus == TStatus::Ok) {
+ CheckExpected(*input);
+ return TStatus::Ok;
+
+ case TExprNode::List:
+ {
+ retStatus = TransformChildren(input, output, ctx);
+ if (retStatus == TStatus::Ok) {
retStatus = CallableTransformer->Transform(input, output, ctx);
if (retStatus == TStatus::Ok) {
input->SetState(TExprNode::EState::ConstrComplete);
@@ -2943,25 +2943,25 @@ private:
}
if (retStatus != TStatus::Error && input != output) {
- processedPair.first->second = output;
- }
- break;
- }
-
- case TExprNode::Lambda:
- {
+ processedPair.first->second = output;
+ }
+ break;
+ }
+
+ case TExprNode::Lambda:
+ {
YQL_ENSURE(input->ChildrenSize() > 0U);
- TExprNode::TPtr out;
+ TExprNode::TPtr out;
auto argStatus = TransformNode(input->HeadPtr(), out, ctx);
UpdateStatusIfChanged(argStatus, input->HeadPtr(), out);
- if (argStatus.Level == TStatus::Error) {
+ if (argStatus.Level == TStatus::Error) {
input->SetState(TExprNode::EState::Error);
- return argStatus;
- }
-
- if (argStatus.Level == TStatus::Repeat)
- return TStatus::Ok;
-
+ return argStatus;
+ }
+
+ if (argStatus.Level == TStatus::Repeat)
+ return TStatus::Ok;
+
TStatus bodyStatus = TStatus::Ok;
TExprNode::TListType newBody;
newBody.reserve(input->ChildrenSize() - 1U);
@@ -2976,144 +2976,144 @@ private:
newBody.emplace_back(std::move(newChild));
}
- retStatus = argStatus.Combine(bodyStatus);
- if (retStatus != TStatus::Ok) {
- if (retStatus.Level == TStatus::Error) {
+ retStatus = argStatus.Combine(bodyStatus);
+ if (retStatus != TStatus::Ok) {
+ if (retStatus.Level == TStatus::Error) {
input->SetState(TExprNode::EState::Error);
- }
+ }
else if (updatedChildren) {
output = ctx.DeepCopyLambda(*input, std::move(newBody));
- processedPair.first->second = output;
- HasRenames = true;
- }
- } else {
+ processedPair.first->second = output;
+ HasRenames = true;
+ }
+ } else {
if (input->ChildrenSize() != 2U)
input->SetState(TExprNode::EState::ConstrComplete);
else
input->CopyConstraints(input->Tail());
- CheckExpected(*input);
- }
- break;
- }
-
- case TExprNode::Argument:
+ CheckExpected(*input);
+ }
+ break;
+ }
+
+ case TExprNode::Argument:
if (input->GetState() != TExprNode::EState::ConstrComplete) {
- return TStatus::Repeat;
- }
-
- return TStatus::Ok;
-
- case TExprNode::Arguments:
- {
- if (input->Children().empty()) {
+ return TStatus::Repeat;
+ }
+
+ return TStatus::Ok;
+
+ case TExprNode::Arguments:
+ {
+ if (input->Children().empty()) {
if (TExprNode::EState::ConstrComplete == input->GetState()) {
- return TStatus::Ok;
- }
- return TStatus::Repeat;
- }
-
- retStatus = TStatus::Ok;
- for (auto& child : input->Children()) {
- TExprNode::TPtr tmp;
- auto childStatus = TransformNode(child, tmp, ctx);
- UpdateStatusIfChanged(childStatus, child, tmp);
- YQL_ENSURE(tmp == child);
- retStatus = retStatus.Combine(childStatus);
- }
-
- if (retStatus != TStatus::Ok) {
- if (retStatus.Level == TStatus::Error) {
+ return TStatus::Ok;
+ }
+ return TStatus::Repeat;
+ }
+
+ retStatus = TStatus::Ok;
+ for (auto& child : input->Children()) {
+ TExprNode::TPtr tmp;
+ auto childStatus = TransformNode(child, tmp, ctx);
+ UpdateStatusIfChanged(childStatus, child, tmp);
+ YQL_ENSURE(tmp == child);
+ retStatus = retStatus.Combine(childStatus);
+ }
+
+ if (retStatus != TStatus::Ok) {
+ if (retStatus.Level == TStatus::Error) {
input->SetState(TExprNode::EState::Error);
- }
- } else {
+ }
+ } else {
input->SetState(TExprNode::EState::ConstrComplete);
- }
- return retStatus;
- }
-
- case TExprNode::Callable:
- {
- retStatus = TransformChildren(input, output, ctx);
- if (retStatus != TStatus::Ok) {
- if (retStatus != TStatus::Error && input != output) {
- processedPair.first->second = output;
- }
- break;
- }
-
- CurrentFunctions.top().second = true;
- retStatus = CallableTransformer->Transform(input, output, ctx);
- if (retStatus == TStatus::Error) {
+ }
+ return retStatus;
+ }
+
+ case TExprNode::Callable:
+ {
+ retStatus = TransformChildren(input, output, ctx);
+ if (retStatus != TStatus::Ok) {
+ if (retStatus != TStatus::Error && input != output) {
+ processedPair.first->second = output;
+ }
+ break;
+ }
+
+ CurrentFunctions.top().second = true;
+ retStatus = CallableTransformer->Transform(input, output, ctx);
+ if (retStatus == TStatus::Error) {
input->SetState(TExprNode::EState::Error);
- } else if (retStatus == TStatus::Ok) {
- // Sanity check
- for (size_t i = 0; i < input->ChildrenSize(); ++i) {
+ } else if (retStatus == TStatus::Ok) {
+ // Sanity check
+ for (size_t i = 0; i < input->ChildrenSize(); ++i) {
YQL_ENSURE(input->Child(i)->GetState() >= TExprNode::EState::ConstrComplete,
"Child with index " << i << " of callable " << TString{input->Content()}.Quote() << " has bad state after constraint transform");
- }
+ }
input->SetState(TExprNode::EState::ConstrComplete);
- CheckExpected(*input);
- } else if (retStatus == TStatus::Async) {
- CallableInputs.push_back(input);
+ CheckExpected(*input);
+ } else if (retStatus == TStatus::Async) {
+ CallableInputs.push_back(input);
input->SetState(TExprNode::EState::ConstrInProgress);
- } else {
- if (output != input.Get()) {
- processedPair.first->second = output;
- HasRenames = true;
- }
- }
- break;
- }
- default:
- YQL_ENSURE(false, "Unknown type");
- }
-
- if (retStatus.Level != TStatus::Repeat || retStatus.HasRestart) {
- return retStatus;
- }
-
- input = output;
- }
- }
-
- TStatus TransformChildren(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- TStatus combinedStatus = TStatus::Ok;
- TExprNode::TListType newChildren;
- bool updatedChildren = false;
- for (ui32 i = 0; i < input->ChildrenSize(); ++i) {
- const auto child = input->ChildPtr(i);
- TExprNode::TPtr newChild;
- auto childStatus = TransformNode(child, newChild, ctx);
- UpdateStatusIfChanged(childStatus, child, newChild);
- updatedChildren = updatedChildren || (newChild != child);
- combinedStatus = combinedStatus.Combine(childStatus);
- newChildren.emplace_back(std::move(newChild));
- }
-
- if (combinedStatus != TStatus::Ok) {
- if (combinedStatus.Level == TStatus::Error) {
+ } else {
+ if (output != input.Get()) {
+ processedPair.first->second = output;
+ HasRenames = true;
+ }
+ }
+ break;
+ }
+ default:
+ YQL_ENSURE(false, "Unknown type");
+ }
+
+ if (retStatus.Level != TStatus::Repeat || retStatus.HasRestart) {
+ return retStatus;
+ }
+
+ input = output;
+ }
+ }
+
+ TStatus TransformChildren(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ TStatus combinedStatus = TStatus::Ok;
+ TExprNode::TListType newChildren;
+ bool updatedChildren = false;
+ for (ui32 i = 0; i < input->ChildrenSize(); ++i) {
+ const auto child = input->ChildPtr(i);
+ TExprNode::TPtr newChild;
+ auto childStatus = TransformNode(child, newChild, ctx);
+ UpdateStatusIfChanged(childStatus, child, newChild);
+ updatedChildren = updatedChildren || (newChild != child);
+ combinedStatus = combinedStatus.Combine(childStatus);
+ newChildren.emplace_back(std::move(newChild));
+ }
+
+ if (combinedStatus != TStatus::Ok) {
+ if (combinedStatus.Level == TStatus::Error) {
input->SetState(TExprNode::EState::Error);
- }
- else if (updatedChildren) {
- output = ctx.ChangeChildren(*input, std::move(newChildren));
- HasRenames = true;
- }
- }
- return combinedStatus;
- }
-
- void UpdateStatusIfChanged(TStatus& status, const TExprNode::TPtr& input, const TExprNode::TPtr& output) {
- if (status.Level == TStatus::Ok && input != output) {
- status = TStatus(TStatus::Repeat, status.HasRestart);
- }
- }
-
- void CheckExpected(const TExprNode& input) {
+ }
+ else if (updatedChildren) {
+ output = ctx.ChangeChildren(*input, std::move(newChildren));
+ HasRenames = true;
+ }
+ }
+ return combinedStatus;
+ }
+
+ void UpdateStatusIfChanged(TStatus& status, const TExprNode::TPtr& input, const TExprNode::TPtr& output) {
+ if (status.Level == TStatus::Ok && input != output) {
+ status = TStatus(TStatus::Repeat, status.HasRestart);
+ }
+ }
+
+ void CheckExpected(const TExprNode& input) {
if (const auto it = Types.ExpectedConstraints.find(input.UniqueId()); it != Types.ExpectedConstraints.cend()) {
- for (const TConstraintNode* expectedConstr: it->second) {
+ for (const TConstraintNode* expectedConstr: it->second) {
if (!Types.DisableConstraintCheck.contains(expectedConstr->GetName())) {
const auto newConstr = input.GetConstraint(expectedConstr->GetName());
- // Constraint Multi(0:{Empty},1:{Empty}, ..., N:{Empty}) can be reduced to Empty
+ // Constraint Multi(0:{Empty},1:{Empty}, ..., N:{Empty}) can be reduced to Empty
if (expectedConstr->GetName() == TMultiConstraintNode::Name()) {
if (newConstr) {
YQL_ENSURE(static_cast<const TMultiConstraintNode*>(newConstr)->FilteredIncludes(*expectedConstr, Types.DisableConstraintCheck), "Rewrite error, unequal " << *newConstr
@@ -3121,123 +3121,123 @@ private:
} else {
YQL_ENSURE(input.GetConstraint<TEmptyConstraintNode>(), "Rewrite error, missing " << *expectedConstr << " constraint in node " << input.Content());
}
- } else {
- YQL_ENSURE(newConstr, "Rewrite error, missing " << *expectedConstr << " constraint in node " << input.Content());
- YQL_ENSURE(newConstr->Includes(*expectedConstr), "Rewrite error, unequal " << *newConstr
- << " constraint in node " << input.Content() << ", previous was " << *expectedConstr);
- }
- }
- }
- }
- }
-
-private:
- TAutoPtr<IGraphTransformer> CallableTransformer;
- TDeque<TExprNode::TPtr> CallableInputs;
- TNodeOnNodeOwnedMap Processed;
- bool HasRenames = false;
- TStack<std::pair<TStringBuf, bool>> CurrentFunctions;
- TTypeAnnotationContext& Types;
-};
-
-} // namespace
-
-TAutoPtr<IGraphTransformer> CreateConstraintTransformer(TTypeAnnotationContext& types, bool instantOnly, bool subGraph) {
- TAutoPtr<IGraphTransformer> callableTransformer(new TCallableConstraintTransformer(types, instantOnly, subGraph));
- return new TConstraintTransformer(callableTransformer, types);
-}
-
-TAutoPtr<IGraphTransformer> CreateDefCallableConstraintTransformer() {
- return new TDefaultCallableConstraintTransformer();
-}
-
-IGraphTransformer::TStatus UpdateLambdaConstraints(const TExprNode& lambda) {
- auto args = lambda.Child(0);
+ } else {
+ YQL_ENSURE(newConstr, "Rewrite error, missing " << *expectedConstr << " constraint in node " << input.Content());
+ YQL_ENSURE(newConstr->Includes(*expectedConstr), "Rewrite error, unequal " << *newConstr
+ << " constraint in node " << input.Content() << ", previous was " << *expectedConstr);
+ }
+ }
+ }
+ }
+ }
+
+private:
+ TAutoPtr<IGraphTransformer> CallableTransformer;
+ TDeque<TExprNode::TPtr> CallableInputs;
+ TNodeOnNodeOwnedMap Processed;
+ bool HasRenames = false;
+ TStack<std::pair<TStringBuf, bool>> CurrentFunctions;
+ TTypeAnnotationContext& Types;
+};
+
+} // namespace
+
+TAutoPtr<IGraphTransformer> CreateConstraintTransformer(TTypeAnnotationContext& types, bool instantOnly, bool subGraph) {
+ TAutoPtr<IGraphTransformer> callableTransformer(new TCallableConstraintTransformer(types, instantOnly, subGraph));
+ return new TConstraintTransformer(callableTransformer, types);
+}
+
+TAutoPtr<IGraphTransformer> CreateDefCallableConstraintTransformer() {
+ return new TDefaultCallableConstraintTransformer();
+}
+
+IGraphTransformer::TStatus UpdateLambdaConstraints(const TExprNode& lambda) {
+ auto args = lambda.Child(0);
for (const auto& arg: args->Children()) {
if (arg->GetState() == TExprNode::EState::TypeComplete || arg->GetState() == TExprNode::EState::ConstrPending) {
arg->SetState(TExprNode::EState::ConstrComplete);
- }
+ }
YQL_ENSURE(arg->GetAllConstraints().empty());
- }
-
+ }
+
if (args->GetState() == TExprNode::EState::TypeComplete || args->GetState() == TExprNode::EState::ConstrPending) {
args->SetState(TExprNode::EState::ConstrComplete);
- }
-
+ }
+
if (lambda.GetState() != TExprNode::EState::ConstrComplete) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- return IGraphTransformer::TStatus::Ok;
-}
-
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ return IGraphTransformer::TStatus::Ok;
+}
+
IGraphTransformer::TStatus UpdateLambdaConstraints(TExprNode::TPtr& lambda, TExprContext& ctx, const TArrayRef<const TConstraintNode::TListType>& constraints) {
- bool updateArgs = false;
+ bool updateArgs = false;
const auto args = lambda->Child(0);
-
- YQL_ENSURE(args->ChildrenSize() == constraints.size());
- size_t i = 0;
+
+ YQL_ENSURE(args->ChildrenSize() == constraints.size());
+ size_t i = 0;
for (const auto constrList: constraints) {
const auto arg = args->Child(i++);
if (arg->GetState() == TExprNode::EState::TypeComplete || arg->GetState() == TExprNode::EState::ConstrPending) {
for (const auto c: constrList) {
- arg->AddConstraint(c);
- }
+ arg->AddConstraint(c);
+ }
arg->SetState(TExprNode::EState::ConstrComplete);
- } else {
+ } else {
if (constrList.size() != arg->GetAllConstraints().size() || !AllOf(constrList, [arg] (const TConstraintNode* c) { return arg->GetConstraint(c->GetName()) == c; })) {
- updateArgs = true;
- }
- }
- }
-
- if (updateArgs) {
+ updateArgs = true;
+ }
+ }
+ }
+
+ if (updateArgs) {
TNodeOnNodeOwnedMap replaces(constraints.size());
TExprNode::TListType argsChildren;
argsChildren.reserve(constraints.size());
- i = 0;
+ i = 0;
for (const auto constrList: constraints) {
const auto arg = args->Child(i++);
const auto newArg = ctx.ShallowCopy(*arg);
- newArg->SetTypeAnn(arg->GetTypeAnn());
+ newArg->SetTypeAnn(arg->GetTypeAnn());
for (const auto c: constrList) {
- newArg->AddConstraint(c);
- }
- newArg->SetState(TExprNode::EState::ConstrComplete);
- YQL_ENSURE(replaces.emplace(arg, newArg).second);
+ newArg->AddConstraint(c);
+ }
+ newArg->SetState(TExprNode::EState::ConstrComplete);
+ YQL_ENSURE(replaces.emplace(arg, newArg).second);
argsChildren.emplace_back(std::move(newArg));
- }
-
- auto newArgs = ctx.NewArguments(args->Pos(), std::move(argsChildren));
- newArgs->SetTypeAnn(ctx.MakeType<TUnitExprType>());
+ }
+
+ auto newArgs = ctx.NewArguments(args->Pos(), std::move(argsChildren));
+ newArgs->SetTypeAnn(ctx.MakeType<TUnitExprType>());
newArgs->SetState(TExprNode::EState::ConstrComplete);
const auto type = lambda->GetTypeAnn();
lambda = ctx.NewLambda(lambda->Pos(), std::move(newArgs), ctx.ReplaceNodes<true>(GetLambdaBody(*lambda), replaces));
lambda->SetTypeAnn(type);
lambda->Head().ForEachChild(std::bind(&TExprNode::SetDependencyScope, std::placeholders::_1, lambda.Get(), lambda.Get()));
return IGraphTransformer::TStatus::Repeat;
- }
-
+ }
+
if (args->GetState() == TExprNode::EState::TypeComplete || args->GetState() == TExprNode::EState::ConstrPending) {
args->SetState(TExprNode::EState::ConstrComplete);
- }
-
+ }
+
if (lambda->GetState() != TExprNode::EState::ConstrComplete) {
- return IGraphTransformer::TStatus::Repeat;
- }
-
- return IGraphTransformer::TStatus::Ok;
-}
-
-IGraphTransformer::TStatus UpdateAllChildLambdasConstraints(const TExprNode& node) {
- IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok;
- for (ui32 i = 0; i < node.ChildrenSize(); ++i) {
- const auto child = node.Child(i);
- if (child->Type() == TExprNode::EType::Lambda) {
- status = status.Combine(UpdateLambdaConstraints(*child));
- }
- }
- return status;
-}
-
-} // namespace NYql
+ return IGraphTransformer::TStatus::Repeat;
+ }
+
+ return IGraphTransformer::TStatus::Ok;
+}
+
+IGraphTransformer::TStatus UpdateAllChildLambdasConstraints(const TExprNode& node) {
+ IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok;
+ for (ui32 i = 0; i < node.ChildrenSize(); ++i) {
+ const auto child = node.Child(i);
+ if (child->Type() == TExprNode::EType::Lambda) {
+ status = status.Combine(UpdateLambdaConstraints(*child));
+ }
+ }
+ return status;
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/core/yql_expr_constraint.h b/ydb/library/yql/core/yql_expr_constraint.h
index 76f64608ab..8e99bbf88a 100644
--- a/ydb/library/yql/core/yql_expr_constraint.h
+++ b/ydb/library/yql/core/yql_expr_constraint.h
@@ -1,20 +1,20 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/core/yql_type_annotation.h>
#include <ydb/library/yql/core/yql_graph_transformer.h>
#include <ydb/library/yql/ast/yql_expr.h>
-
-#include <util/generic/ptr.h>
-
-#include <initializer_list>
-
-namespace NYql {
-
-TAutoPtr<IGraphTransformer> CreateConstraintTransformer(TTypeAnnotationContext& types, bool instantOnly = false, bool subGraph = false);
-TAutoPtr<IGraphTransformer> CreateDefCallableConstraintTransformer();
-
-IGraphTransformer::TStatus UpdateLambdaConstraints(const TExprNode& lambda);
+
+#include <util/generic/ptr.h>
+
+#include <initializer_list>
+
+namespace NYql {
+
+TAutoPtr<IGraphTransformer> CreateConstraintTransformer(TTypeAnnotationContext& types, bool instantOnly = false, bool subGraph = false);
+TAutoPtr<IGraphTransformer> CreateDefCallableConstraintTransformer();
+
+IGraphTransformer::TStatus UpdateLambdaConstraints(const TExprNode& lambda);
IGraphTransformer::TStatus UpdateLambdaConstraints(TExprNode::TPtr& lambda, TExprContext& ctx, const TArrayRef<const TConstraintNode::TListType>& constraints);
-IGraphTransformer::TStatus UpdateAllChildLambdasConstraints(const TExprNode& node);
-
-}
+IGraphTransformer::TStatus UpdateAllChildLambdasConstraints(const TExprNode& node);
+
+}
diff --git a/ydb/library/yql/core/yql_expr_csee.cpp b/ydb/library/yql/core/yql_expr_csee.cpp
index b0e7525521..5d1d33e7ed 100644
--- a/ydb/library/yql/core/yql_expr_csee.cpp
+++ b/ydb/library/yql/core/yql_expr_csee.cpp
@@ -112,11 +112,11 @@ namespace {
break;
}
- ui64 hash = node.GetTypeAnn()->GetHash();
+ ui64 hash = node.GetTypeAnn()->GetHash();
hash = CseeHash(ui32(node.Type()), hash);
- for (auto c: node.GetAllConstraints()) {
+ for (auto c: node.GetAllConstraints()) {
hash = CseeHash(c->GetHash(), hash);
- }
+ }
hash = AddColumnOrderHash(coStore.Lookup(node.UniqueId()), hash);
switch (node.Type()) {
@@ -173,9 +173,9 @@ namespace {
for (ui32 i = 0; i < args.ChildrenSize(); ++i) {
const auto& arg = *args.Child(i);
hash = CseeHash(arg.GetTypeAnn()->GetHash(), hash);
- for (auto c: arg.GetAllConstraints()) {
+ for (auto c: arg.GetAllConstraints()) {
hash = CseeHash(c->GetHash(), hash);
- }
+ }
}
TLambdaFrame newFrame(&node, &currFrame);
@@ -247,14 +247,14 @@ namespace {
return false;
}
- if (left.GetTypeAnn() != right.GetTypeAnn()) {
- return false;
- }
-
- if (left.GetAllConstraints() != right.GetAllConstraints()) {
+ if (left.GetTypeAnn() != right.GetTypeAnn()) {
return false;
}
+ if (left.GetAllConstraints() != right.GetAllConstraints()) {
+ return false;
+ }
+
if (coStore.Lookup(left.UniqueId()) != coStore.Lookup(right.UniqueId())) {
return false;
}
@@ -326,12 +326,12 @@ namespace {
for (ui32 i = 0; i < leftArgs.ChildrenSize(); ++i) {
const auto& leftArg = *leftArgs.Child(i);
const auto& rightArg = *rightArgs.Child(i);
- if (leftArg.GetTypeAnn() != rightArg.GetTypeAnn()) {
- return false;
- }
- if (leftArg.GetAllConstraints() != rightArg.GetAllConstraints()) {
+ if (leftArg.GetTypeAnn() != rightArg.GetTypeAnn()) {
return false;
}
+ if (leftArg.GetAllConstraints() != rightArg.GetAllConstraints()) {
+ return false;
+ }
}
TLambdaFrame newLeftFrame(&left, &currLeftFrame);
diff --git a/ydb/library/yql/core/yql_expr_optimize.cpp b/ydb/library/yql/core/yql_expr_optimize.cpp
index 677ca7fa53..89d6f7e0f8 100644
--- a/ydb/library/yql/core/yql_expr_optimize.cpp
+++ b/ydb/library/yql/core/yql_expr_optimize.cpp
@@ -12,41 +12,41 @@ namespace {
TExprContext& Expr;
const TOptimizeExprSettings& Settings;
TNodeOnNodeOwnedMap Memoization;
- const TNodeOnNodeOwnedMap* Replaces;
+ const TNodeOnNodeOwnedMap* Replaces;
ui64 LastNodeId;
bool HasRemaps;
- TOptimizationContext(TOptimizer optimizer, const TNodeOnNodeOwnedMap* replaces, TExprContext& expr, const TOptimizeExprSettings& settings)
+ TOptimizationContext(TOptimizer optimizer, const TNodeOnNodeOwnedMap* replaces, TExprContext& expr, const TOptimizeExprSettings& settings)
: Optimizer(optimizer)
, Expr(expr)
, Settings(settings)
- , Replaces(replaces)
+ , Replaces(replaces)
, LastNodeId(expr.NextUniqueId)
- , HasRemaps(false)
- {
- }
+ , HasRemaps(false)
+ {
+ }
void RemapNode(const TExprNode& fromNode, const TExprNode::TPtr& toNode) final {
YQL_ENSURE(fromNode.UniqueId() <= LastNodeId);
YQL_ENSURE(toNode->UniqueId() > LastNodeId);
Memoization[&fromNode] = toNode;
HasRemaps = true;
- if (Settings.ProcessedNodes) {
- Settings.ProcessedNodes->erase(fromNode.UniqueId());
- }
+ if (Settings.ProcessedNodes) {
+ Settings.ProcessedNodes->erase(fromNode.UniqueId());
+ }
}
};
TExprNode::TPtr RunOptimizer(TOptimizationContext<TCallableOptimizer>& ctx, const TExprNode::TPtr& node) {
- return (!ctx.Settings.VisitChanges && ctx.HasRemaps) ? node : ctx.Optimizer(node, ctx.Expr);
+ return (!ctx.Settings.VisitChanges && ctx.HasRemaps) ? node : ctx.Optimizer(node, ctx.Expr);
}
TExprNode::TPtr RunOptimizer(TOptimizationContext<TCallableOptimizerEx>& ctx, const TExprNode::TPtr& node) {
- return (!ctx.Settings.VisitChanges && ctx.HasRemaps) ? node : ctx.Optimizer(node, ctx.Expr, ctx);
+ return (!ctx.Settings.VisitChanges && ctx.HasRemaps) ? node : ctx.Optimizer(node, ctx.Expr, ctx);
}
TExprNode::TPtr RunOptimizer(TOptimizationContext<TCallableOptimizerFast>& ctx, bool& changed, const TExprNode::TPtr& node) {
- return (!ctx.Settings.VisitChanges && ctx.HasRemaps) ? node : ctx.Optimizer(node, changed, ctx.Expr);
+ return (!ctx.Settings.VisitChanges && ctx.HasRemaps) ? node : ctx.Optimizer(node, changed, ctx.Expr);
}
template<typename TContext>
@@ -72,8 +72,8 @@ namespace {
return hasRemaps ? ctx.Expr.ChangeChildren(*node, std::move(newChildren)) : node;
}
- void AddExpected(const TExprNode& src, const TExprNode& dst, const TOptimizeExprSettings& settings) {
- if (!src.GetTypeAnn() || dst.GetTypeAnn() || !settings.Types) {
+ void AddExpected(const TExprNode& src, const TExprNode& dst, const TOptimizeExprSettings& settings) {
+ if (!src.GetTypeAnn() || dst.GetTypeAnn() || !settings.Types) {
return;
}
@@ -81,8 +81,8 @@ namespace {
return;
}
- settings.Types->ExpectedTypes.emplace(dst.UniqueId(), src.GetTypeAnn());
- settings.Types->ExpectedConstraints.emplace(dst.UniqueId(), src.GetAllConstraints());
+ settings.Types->ExpectedTypes.emplace(dst.UniqueId(), src.GetTypeAnn());
+ settings.Types->ExpectedConstraints.emplace(dst.UniqueId(), src.GetAllConstraints());
auto columnOrder = settings.Types->LookupColumnOrder(src);
if (columnOrder) {
settings.Types->ExpectedColumnOrders.emplace(dst.UniqueId(), *columnOrder);
@@ -113,17 +113,17 @@ namespace {
return it->second ? it->second : node;
}
- TExprNode::TPtr current = node;
- if (ctx.Replaces) {
- const auto it = ctx.Replaces->find(node.Get());
- if (it != ctx.Replaces->cend()) {
- current = it->second;
- }
- }
-
+ TExprNode::TPtr current = node;
+ if (ctx.Replaces) {
+ const auto it = ctx.Replaces->find(node.Get());
+ if (it != ctx.Replaces->cend()) {
+ current = it->second;
+ }
+ }
+
TExprNode::TPtr ret;
- if (current->Type() == TExprNode::Lambda) {
- ret = current;
+ if (current->Type() == TExprNode::Lambda) {
+ ret = current;
if (ctx.Settings.VisitLambdas) {
TExprNode::TListType newBody;
@@ -144,14 +144,14 @@ namespace {
}
if (bodyChanged) {
ret = ctx.Expr.DeepCopyLambda(*current, std::move(newBody));
- AddExpected(*node, *ret, ctx.Settings);
+ AddExpected(*node, *ret, ctx.Settings);
}
}
} else {
TExprNode::TListType newChildren;
- newChildren.reserve(current->ChildrenSize());
+ newChildren.reserve(current->ChildrenSize());
bool hasRenames = false;
- for (auto& child : current->Children()) {
+ for (auto& child : current->Children()) {
auto newChild = OptimizeNode(child, ctx, level + 1);
if (!newChild)
return nullptr;
@@ -168,18 +168,18 @@ namespace {
}
}
- auto renamedNode = hasRenames ? ctx.Expr.ChangeChildren(*current, std::move(newChildren)) : TExprNode::TPtr();
+ auto renamedNode = hasRenames ? ctx.Expr.ChangeChildren(*current, std::move(newChildren)) : TExprNode::TPtr();
newChildren.clear();
if (!ctx.Settings.VisitChanges && hasRenames && ctx.Settings.CustomInstantTypeTransformer) {
- auto root = renamedNode ? renamedNode : current;
+ auto root = renamedNode ? renamedNode : current;
ctx.Settings.CustomInstantTypeTransformer->Rewind();
auto status = InstantTransform(*ctx.Settings.CustomInstantTypeTransformer, root, ctx.Expr);
if (status.Level == IGraphTransformer::TStatus::Error) {
return nullptr;
}
- YQL_ENSURE(root->GetTypeAnn());
+ YQL_ENSURE(root->GetTypeAnn());
if (status.HasRestart) {
ret = std::move(root);
} else {
@@ -189,9 +189,9 @@ namespace {
}
if (!ret) {
- const auto& nextNode = renamedNode ? renamedNode : current;
- const bool visitTuples = ctx.Settings.VisitTuples && nextNode->Type() == TExprNode::List;
- if ((nextNode->Type() != TExprNode::Callable && !visitTuples) || (hasRenames && !ctx.Settings.VisitChanges) || !ctx.Optimizer) {
+ const auto& nextNode = renamedNode ? renamedNode : current;
+ const bool visitTuples = ctx.Settings.VisitTuples && nextNode->Type() == TExprNode::List;
+ if ((nextNode->Type() != TExprNode::Callable && !visitTuples) || (hasRenames && !ctx.Settings.VisitChanges) || !ctx.Optimizer) {
ret = nextNode;
} else {
ret = RunOptimizer(ctx, nextNode);
@@ -200,7 +200,7 @@ namespace {
}
}
- AddExpected(*node, *ret, ctx.Settings);
+ AddExpected(*node, *ret, ctx.Settings);
}
if (node == ret && ctx.Settings.ProcessedNodes) {
@@ -209,13 +209,13 @@ namespace {
if (node == ret) {
YQL_ENSURE(ctx.Memoization.emplace(node.Get(), TExprNode::TPtr()).second);
- } else {
- if (!node->Unique()) {
- YQL_ENSURE(ctx.Memoization.emplace(node.Get(), ret).second);
- }
- if (current != node && current != ret) {
- ctx.Memoization.emplace(current.Get(), ret);
- }
+ } else {
+ if (!node->Unique()) {
+ YQL_ENSURE(ctx.Memoization.emplace(node.Get(), ret).second);
+ }
+ if (current != node && current != ret) {
+ ctx.Memoization.emplace(current.Get(), ret);
+ }
}
return ret;
}
@@ -286,7 +286,7 @@ namespace {
changed = changed || hasRenames;
const auto& nextNode = renamedNode ? renamedNode : node;
- if (nextNode->Type() != TExprNode::Callable && (nextNode->Type() != TExprNode::List || !ctx.Settings.VisitTuples)) {
+ if (nextNode->Type() != TExprNode::Callable && (nextNode->Type() != TExprNode::List || !ctx.Settings.VisitTuples)) {
ret = nextNode;
} else {
bool optimized = false;
@@ -339,56 +339,56 @@ namespace {
}
}
- void VisitExprByFirstInternal(const TExprNode::TPtr& node, const TExprVisitPtrFunc& preFunc,
- const TExprVisitPtrFunc& postFunc, TNodeSet& visitedNodes)
- {
+ void VisitExprByFirstInternal(const TExprNode::TPtr& node, const TExprVisitPtrFunc& preFunc,
+ const TExprVisitPtrFunc& postFunc, TNodeSet& visitedNodes)
+ {
if (!visitedNodes.emplace(node.Get()).second) {
- return;
- }
-
+ return;
+ }
+
if (!preFunc || preFunc(node)) {
- if (node->ChildrenSize() > 0) {
- if (node->Content() == SyncName) {
- for (const auto& child : node->Children()) {
- VisitExprByFirstInternal(child, preFunc, postFunc, visitedNodes);
- }
- }
- else {
+ if (node->ChildrenSize() > 0) {
+ if (node->Content() == SyncName) {
+ for (const auto& child : node->Children()) {
+ VisitExprByFirstInternal(child, preFunc, postFunc, visitedNodes);
+ }
+ }
+ else {
VisitExprByFirstInternal(node->HeadPtr(), preFunc, postFunc, visitedNodes);
- }
- }
- }
-
- if (postFunc) {
- postFunc(node);
- }
- }
-
- void VisitExprByFirstInternal(const TExprNode& node, const TExprVisitRefFunc& preFunc,
- const TExprVisitRefFunc& postFunc, TNodeSet& visitedNodes)
- {
+ }
+ }
+ }
+
+ if (postFunc) {
+ postFunc(node);
+ }
+ }
+
+ void VisitExprByFirstInternal(const TExprNode& node, const TExprVisitRefFunc& preFunc,
+ const TExprVisitRefFunc& postFunc, TNodeSet& visitedNodes)
+ {
if (!visitedNodes.emplace(&node).second) {
- return;
- }
-
+ return;
+ }
+
if (!preFunc || preFunc(node)) {
- if (node.ChildrenSize() > 0) {
- if (node.Content() == SyncName) {
- for (const auto& child : node.Children()) {
- VisitExprByFirstInternal(*child, preFunc, postFunc, visitedNodes);
- }
- }
- else {
+ if (node.ChildrenSize() > 0) {
+ if (node.Content() == SyncName) {
+ for (const auto& child : node.Children()) {
+ VisitExprByFirstInternal(*child, preFunc, postFunc, visitedNodes);
+ }
+ }
+ else {
VisitExprByFirstInternal(node.Head(), preFunc, postFunc, visitedNodes);
- }
- }
- }
-
- if (postFunc) {
- postFunc(node);
- }
- }
-
+ }
+ }
+ }
+
+ if (postFunc) {
+ postFunc(node);
+ }
+ }
+
void VisitExprByPrimaryBranch(const TExprNode::TPtr& node, const TExprVisitPtrFunc& predicate, bool& primary, TNodeSet& visitedNodes)
{
if (!visitedNodes.emplace(node.Get()).second) {
@@ -424,9 +424,9 @@ namespace {
if (optCtx.HasRemaps) {
output = ApplyRemaps(output, optCtx);
- if (settings.ProcessedNodes) {
- settings.ProcessedNodes->clear();
- }
+ if (settings.ProcessedNodes) {
+ settings.ProcessedNodes->clear();
+ }
}
if (!settings.VisitChanges && (output != input)) {
@@ -463,13 +463,13 @@ namespace {
IGraphTransformer::TStatus OptimizeExpr(const TExprNode::TPtr& input, TExprNode::TPtr& output, TCallableOptimizer optimizer,
TExprContext& ctx, const TOptimizeExprSettings& settings)
{
- return OptimizeExprInternal(input, output, optimizer, nullptr, ctx, settings);
+ return OptimizeExprInternal(input, output, optimizer, nullptr, ctx, settings);
}
IGraphTransformer::TStatus OptimizeExprEx(const TExprNode::TPtr& input, TExprNode::TPtr& output, TCallableOptimizerEx optimizer,
TExprContext& ctx, const TOptimizeExprSettings& settings)
{
- return OptimizeExprInternal(input, output, optimizer, nullptr, ctx, settings);
+ return OptimizeExprInternal(input, output, optimizer, nullptr, ctx, settings);
}
IGraphTransformer::TStatus OptimizeExpr(const TExprNode::TPtr& input, TExprNode::TPtr& output, const TCallableOptimizerFast& optimizer,
@@ -478,12 +478,12 @@ IGraphTransformer::TStatus OptimizeExpr(const TExprNode::TPtr& input, TExprNode:
return OptimizeExprInternal(input, output, optimizer, ctx, settings);
}
-IGraphTransformer::TStatus RemapExpr(const TExprNode::TPtr& input, TExprNode::TPtr& output, const TNodeOnNodeOwnedMap& remaps,
- TExprContext& ctx, const TOptimizeExprSettings& settings)
-{
- return OptimizeExprInternal<TCallableOptimizer>(input, output, {}, &remaps, ctx, settings);
-}
-
+IGraphTransformer::TStatus RemapExpr(const TExprNode::TPtr& input, TExprNode::TPtr& output, const TNodeOnNodeOwnedMap& remaps,
+ TExprContext& ctx, const TOptimizeExprSettings& settings)
+{
+ return OptimizeExprInternal<TCallableOptimizer>(input, output, {}, &remaps, ctx, settings);
+}
+
IGraphTransformer::TStatus ExpandApply(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
if (ctx.Step.IsDone(TExprStep::ExpandApplyForLambdas))
return IGraphTransformer::TStatus::Ok;
@@ -827,13 +827,13 @@ TExprNode::TPtr ApplySyncListToWorld(const TExprNode::TPtr& main, const TSyncMap
syncChildren.push_back(main);
Sort(sortedList, [](const TPair& x, const TPair& y) { return x.second < y.second; });
for (auto x : sortedList) {
- if (x.first->IsCallable(RightName)) {
+ if (x.first->IsCallable(RightName)) {
auto world = ctx.NewCallable(main->Pos(), LeftName, { x.first->HeadPtr() });
syncChildren.push_back(world);
- } else if (x.first->GetTypeAnn()->GetKind() == ETypeAnnotationKind::World) {
+ } else if (x.first->GetTypeAnn()->GetKind() == ETypeAnnotationKind::World) {
syncChildren.push_back(x.first);
} else {
- auto world = ctx.NewCallable(main->Pos(), LeftName, { x.first });
+ auto world = ctx.NewCallable(main->Pos(), LeftName, { x.first });
syncChildren.push_back(world);
}
}
@@ -860,25 +860,25 @@ void VisitExpr(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func, TNode
VisitExprInternal(root, func, {}, visitedNodes);
}
-void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func) {
- TNodeSet visitedNodes;
- VisitExprByFirstInternal(root, func, {}, visitedNodes);
-}
-
-void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& preFunc, const TExprVisitPtrFunc& postFunc) {
- TNodeSet visitedNodes;
- VisitExprByFirstInternal(root, preFunc, postFunc, visitedNodes);
-}
-
-void VisitExprByFirst(const TExprNode& root, const TExprVisitRefFunc& func) {
- TNodeSet visitedNodes;
- VisitExprByFirstInternal(root, func, {}, visitedNodes);
-}
-
-void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func, TNodeSet& visitedNodes) {
- VisitExprByFirstInternal(root, func, {}, visitedNodes);
-}
-
+void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func) {
+ TNodeSet visitedNodes;
+ VisitExprByFirstInternal(root, func, {}, visitedNodes);
+}
+
+void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& preFunc, const TExprVisitPtrFunc& postFunc) {
+ TNodeSet visitedNodes;
+ VisitExprByFirstInternal(root, preFunc, postFunc, visitedNodes);
+}
+
+void VisitExprByFirst(const TExprNode& root, const TExprVisitRefFunc& func) {
+ TNodeSet visitedNodes;
+ VisitExprByFirstInternal(root, func, {}, visitedNodes);
+}
+
+void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func, TNodeSet& visitedNodes) {
+ VisitExprByFirstInternal(root, func, {}, visitedNodes);
+}
+
TExprNode::TPtr FindNode(const TExprNode::TPtr& root, const TExprVisitPtrFunc& predicate) {
TExprNode::TPtr result;
VisitExpr(root, [&result, &predicate] (const TExprNode::TPtr& node) {
diff --git a/ydb/library/yql/core/yql_expr_optimize.h b/ydb/library/yql/core/yql_expr_optimize.h
index ae5f0dea38..fed51d2dd3 100644
--- a/ydb/library/yql/core/yql_expr_optimize.h
+++ b/ydb/library/yql/core/yql_expr_optimize.h
@@ -23,7 +23,7 @@ struct TOptimizeExprSettings {
IGraphTransformer* CustomInstantTypeTransformer = nullptr;
bool VisitLambdas = true;
TTypeAnnotationContext* Types;
- bool VisitTuples = false;
+ bool VisitTuples = false;
};
IGraphTransformer::TStatus OptimizeExpr(const TExprNode::TPtr& input, TExprNode::TPtr& output, TCallableOptimizer optimizer,
@@ -32,10 +32,10 @@ IGraphTransformer::TStatus OptimizeExpr(const TExprNode::TPtr& input, TExprNode:
IGraphTransformer::TStatus OptimizeExpr(const TExprNode::TPtr& input, TExprNode::TPtr& output, const TCallableOptimizerFast& optimizer,
TExprContext& ctx, const TOptimizeExprSettings& settings);
-IGraphTransformer::TStatus RemapExpr(const TExprNode::TPtr& input, TExprNode::TPtr& output, const TNodeOnNodeOwnedMap& remaps,
- TExprContext& ctx, const TOptimizeExprSettings& settings);
-
-
+IGraphTransformer::TStatus RemapExpr(const TExprNode::TPtr& input, TExprNode::TPtr& output, const TNodeOnNodeOwnedMap& remaps,
+ TExprContext& ctx, const TOptimizeExprSettings& settings);
+
+
class IOptimizationContext {
public:
virtual ~IOptimizationContext() = default;
@@ -58,11 +58,11 @@ void VisitExpr(const TExprNode::TPtr& root, const TExprVisitPtrFunc& preFunc, co
void VisitExpr(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func, TNodeSet& visitedNodes);
void VisitExpr(const TExprNode& root, const TExprVisitRefFunc& func);
-void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func);
-void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& preFunc, const TExprVisitPtrFunc& postFunc);
-void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func, TNodeSet& visitedNodes);
-void VisitExprByFirst(const TExprNode& root, const TExprVisitRefFunc& func);
-
+void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func);
+void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& preFunc, const TExprVisitPtrFunc& postFunc);
+void VisitExprByFirst(const TExprNode::TPtr& root, const TExprVisitPtrFunc& func, TNodeSet& visitedNodes);
+void VisitExprByFirst(const TExprNode& root, const TExprVisitRefFunc& func);
+
TExprNode::TPtr FindNode(const TExprNode::TPtr& root, const TExprVisitPtrFunc& predicate);
TExprNode::TPtr FindNode(const TExprNode::TPtr& root, const TExprVisitPtrFunc& filter, const TExprVisitPtrFunc& predicate);
diff --git a/ydb/library/yql/core/yql_expr_type_annotation.cpp b/ydb/library/yql/core/yql_expr_type_annotation.cpp
index f2b793af8d..6317e5a445 100644
--- a/ydb/library/yql/core/yql_expr_type_annotation.cpp
+++ b/ydb/library/yql/core/yql_expr_type_annotation.cpp
@@ -10,7 +10,7 @@
#include <ydb/library/yql/public/decimal/yql_decimal.h>
#include <ydb/library/yql/utils/yql_panic.h>
#include <ydb/library/yql/utils/utf8.h>
-
+
#include <util/generic/hash_set.h>
#include <util/generic/algorithm.h>
#include <util/string/builder.h>
@@ -23,7 +23,7 @@
namespace NYql {
using namespace NNodes;
-using namespace NKikimr;
+using namespace NKikimr;
namespace {
@@ -990,16 +990,16 @@ NUdf::TCastResultOptions CastResult(const TListExprType* source, const TListExpr
}
template <bool Strong>
-NUdf::TCastResultOptions CastResult(const TStreamExprType* source, const TStreamExprType* target) {
- return ReduceCastResult<Strong>(CastResult<Strong>(source->GetItemType(), target->GetItemType()));
-}
-
-template <bool Strong>
-NUdf::TCastResultOptions CastResult(const TFlowExprType* source, const TFlowExprType* target) {
- return ReduceCastResult<Strong>(CastResult<Strong>(source->GetItemType(), target->GetItemType()));
-}
-
-template <bool Strong>
+NUdf::TCastResultOptions CastResult(const TStreamExprType* source, const TStreamExprType* target) {
+ return ReduceCastResult<Strong>(CastResult<Strong>(source->GetItemType(), target->GetItemType()));
+}
+
+template <bool Strong>
+NUdf::TCastResultOptions CastResult(const TFlowExprType* source, const TFlowExprType* target) {
+ return ReduceCastResult<Strong>(CastResult<Strong>(source->GetItemType(), target->GetItemType()));
+}
+
+template <bool Strong>
NUdf::TCastResultOptions CastResult(const TDictExprType* source, const TDictExprType* target) {
return ReduceCastResult<Strong>(CastResult<Strong>(source->GetKeyType(), target->GetKeyType()) | CastResult<Strong>(source->GetPayloadType(), target->GetPayloadType()));
}
@@ -1473,12 +1473,12 @@ NUdf::TCastResultOptions CastResult(const TTypeAnnotationNode* source, const TTy
return CastResult<Strong>(source->Cast<TVariantExprType>(), target->Cast<TVariantExprType>());
case ETypeAnnotationKind::Data:
return CastResult<Strong>(source->Cast<TDataExprType>(), target->Cast<TDataExprType>());
- case ETypeAnnotationKind::Stream:
- return CastResult<Strong>(source->Cast<TStreamExprType>(), target->Cast<TStreamExprType>());
- case ETypeAnnotationKind::Flow:
- return CastResult<Strong>(source->Cast<TFlowExprType>(), target->Cast<TFlowExprType>());
- case ETypeAnnotationKind::Resource:
- return source->Cast<TResourceExprType>()->GetTag() == target->Cast<TResourceExprType>()->GetTag() ? NUdf::ECastOptions::Complete : NUdf::ECastOptions::Impossible;
+ case ETypeAnnotationKind::Stream:
+ return CastResult<Strong>(source->Cast<TStreamExprType>(), target->Cast<TStreamExprType>());
+ case ETypeAnnotationKind::Flow:
+ return CastResult<Strong>(source->Cast<TFlowExprType>(), target->Cast<TFlowExprType>());
+ case ETypeAnnotationKind::Resource:
+ return source->Cast<TResourceExprType>()->GetTag() == target->Cast<TResourceExprType>()->GetTag() ? NUdf::ECastOptions::Complete : NUdf::ECastOptions::Impossible;
default: break;
}
} else if (sKind == ETypeAnnotationKind::Null) {
@@ -1709,11 +1709,11 @@ void ClearExprTypeAnnotations(TExprNode& root) {
ClearExprTypeAnnotations(*child);
}
- root.SetTypeAnn(nullptr);
+ root.SetTypeAnn(nullptr);
}
bool AreAllNodesTypeAnnotated(const TExprNode& root) {
- if (!root.GetTypeAnn())
+ if (!root.GetTypeAnn())
return false;
for (auto& child : root.Children()) {
@@ -1725,7 +1725,7 @@ bool AreAllNodesTypeAnnotated(const TExprNode& root) {
}
void EnsureAllNodesTypeAnnotated(const TExprNode& root) {
- YQL_ENSURE(root.GetTypeAnn());
+ YQL_ENSURE(root.GetTypeAnn());
for (auto& child : root.Children()) {
EnsureAllNodesTypeAnnotated(*child);
}
@@ -1842,7 +1842,7 @@ bool EnsureCallableMaxArgsCount(const TPositionHandle& pos, ui32 args, ui32 expe
}
bool EnsureAtom(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::Atom) {
+ if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::Atom) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected atom, but got: " << node.Type()));
return false;
}
@@ -1851,7 +1851,7 @@ bool EnsureAtom(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureCallable(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::Callable) {
+ if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::Callable) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected callable, but got: " << node.Type()));
return false;
}
@@ -1860,7 +1860,7 @@ bool EnsureCallable(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureTuple(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) {
+ if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type()));
return false;
}
@@ -1882,7 +1882,7 @@ bool EnsureTupleOfAtoms(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureTupleSize(const TExprNode& node, ui32 expectedSize, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) {
+ if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type()));
return false;
}
@@ -1897,7 +1897,7 @@ bool EnsureTupleSize(const TExprNode& node, ui32 expectedSize, TExprContext& ctx
}
bool EnsureTupleMinSize(const TExprNode& node, ui32 minSize, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) {
+ if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type()));
return false;
}
@@ -1912,7 +1912,7 @@ bool EnsureTupleMinSize(const TExprNode& node, ui32 minSize, TExprContext& ctx)
}
bool EnsureTupleMaxSize(const TExprNode& node, ui32 maxSize, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) {
+ if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::List) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple, but got: " << node.Type()));
return false;
}
@@ -1927,13 +1927,13 @@ bool EnsureTupleMaxSize(const TExprNode& node, ui32 maxSize, TExprContext& ctx)
}
bool EnsureTupleType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tuple) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tuple) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tuple type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2001,13 +2001,13 @@ bool EnsureMultiType(TPositionHandle position, const TTypeAnnotationNode& type,
}
bool EnsureVariantType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected variant type, but got lambda"));
return false;
}
- return EnsureVariantType(node.Pos(), *node.GetTypeAnn(), ctx);
+ return EnsureVariantType(node.Pos(), *node.GetTypeAnn(), ctx);
}
bool EnsureVariantType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
@@ -2020,13 +2020,13 @@ bool EnsureVariantType(TPositionHandle position, const TTypeAnnotationNode& type
}
bool EnsureDataType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got lambda"));
return false;
}
- return EnsureDataType(node.Pos(), *node.GetTypeAnn(), ctx);
+ return EnsureDataType(node.Pos(), *node.GetTypeAnn(), ctx);
}
bool EnsureDataType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
@@ -2039,7 +2039,7 @@ bool EnsureDataType(TPositionHandle position, const TTypeAnnotationNode& type, T
}
bool EnsureLambda(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::Lambda) {
+ if (HasError(node.GetTypeAnn(), ctx) || node.Type() != TExprNode::Lambda) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected lambda, but got: " << node.Type()));
return false;
}
@@ -2122,11 +2122,11 @@ IGraphTransformer::TStatus ConvertToLambda(TExprNode::TPtr& node, TExprContext&
return IGraphTransformer::TStatus::Error;
}
- auto callableType = node->GetTypeAnn()->Cast<TCallableExprType>();
+ auto callableType = node->GetTypeAnn()->Cast<TCallableExprType>();
if (minArgumentsCount != Max<ui32>() && (minArgumentsCount > callableType->GetArgumentsSize() ||
minArgumentsCount < callableType->GetArgumentsSize() - callableType->GetOptionalArgumentsCount())) {
ctx.AddError(TIssue(ctx.GetPosition(node->Pos()), TStringBuilder() << "Failed to convert to lambda with "
- << minArgumentsCount << " arguments from callable type " << *node->GetTypeAnn()));
+ << minArgumentsCount << " arguments from callable type " << *node->GetTypeAnn()));
return IGraphTransformer::TStatus::Error;
}
@@ -2145,7 +2145,7 @@ IGraphTransformer::TStatus ConvertToLambda(TExprNode::TPtr& node, TExprContext&
}
bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, TExprContext& ctx, bool allowOptional) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got lambda"));
return false;
@@ -2156,15 +2156,15 @@ bool EnsureSpecificDataType(const TExprNode& node, EDataSlot expectedDataSlot, T
return EnsureSpecificDataType(node.Pos(), *optionalType->GetItemType(), expectedDataSlot, ctx);
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Data) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type, but got: " << *node.GetTypeAnn()));
return false;
}
- auto dataSlot = node.GetTypeAnn()->Cast<TDataExprType>()->GetSlot();
+ auto dataSlot = node.GetTypeAnn()->Cast<TDataExprType>()->GetSlot();
if (dataSlot != expectedDataSlot) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected data type: " << NKikimr::NUdf::GetDataTypeInfo(expectedDataSlot).Name << ", but got: " <<
- *node.GetTypeAnn()));
+ *node.GetTypeAnn()));
return false;
}
@@ -2222,13 +2222,13 @@ bool EnsureStringOrUtf8Type(TPositionHandle position, const TTypeAnnotationNode&
}
bool EnsureStructType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected struct type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected struct type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2267,13 +2267,13 @@ bool EnsureComposable(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureComposableType(const TExprNode& node, TExprContext& ctx) {
- if (!node.GetTypeAnn()) {
+ if (!node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected composable type, but got lambda"));
return false;
}
- return EnsureComposableType(node.Pos(), *node.GetTypeAnn(), ctx);
+ return EnsureComposableType(node.Pos(), *node.GetTypeAnn(), ctx);
}
bool EnsureComposableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
@@ -2286,13 +2286,13 @@ bool EnsureComposableType(TPositionHandle position, const TTypeAnnotationNode& t
}
bool EnsureWorldType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected world type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::World) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected world type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2301,7 +2301,7 @@ bool EnsureWorldType(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureDataSource(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource callable, but got lambda"));
return false;
@@ -2316,7 +2316,7 @@ bool EnsureDataSource(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureDataSink(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink callable, but got lambda"));
return false;
@@ -2331,7 +2331,7 @@ bool EnsureDataSink(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureDataProvider(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource or datasink callable, but got lambda"));
return false;
@@ -2346,7 +2346,7 @@ bool EnsureDataProvider(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureSpecificDataSource(const TExprNode& node, TStringBuf expectedCategory, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasource callable, but got lambda"));
return false;
@@ -2368,7 +2368,7 @@ bool EnsureSpecificDataSource(const TExprNode& node, TStringBuf expectedCategory
}
bool EnsureSpecificDataSink(const TExprNode& node, TStringBuf expectedCategory, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected datasink callable, but got lambda"));
return false;
@@ -2390,13 +2390,13 @@ bool EnsureSpecificDataSink(const TExprNode& node, TStringBuf expectedCategory,
}
bool EnsureListType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::List) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::List) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2439,13 +2439,13 @@ bool EnsureListOrEmptyType(TPositionHandle position, const TTypeAnnotationNode&
}
bool EnsureListOfVoidType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list of void type, but got lambda"));
return false;
}
- return EnsureListOfVoidType(node.Pos(), *node.GetTypeAnn(), ctx);
+ return EnsureListOfVoidType(node.Pos(), *node.GetTypeAnn(), ctx);
}
bool EnsureListOfVoidType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
@@ -2465,13 +2465,13 @@ bool EnsureListOfVoidType(TPositionHandle position, const TTypeAnnotationNode& t
}
bool EnsureStreamType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected stream type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Stream) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Stream) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected stream type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2489,13 +2489,13 @@ bool EnsureStreamType(TPositionHandle position, const TTypeAnnotationNode& type,
}
bool EnsureFlowType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected flow type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Flow) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Flow) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected flow type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2537,13 +2537,13 @@ bool EnsureWideFlowType(TPositionHandle position, const TTypeAnnotationNode& typ
}
bool EnsureOptionalType(const TExprNode& node, TExprContext& ctx) {
- if (!node.GetTypeAnn()) {
+ if (!node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected optional type, but got lambda"));
return false;
}
- return EnsureOptionalType(node.Pos(), *node.GetTypeAnn(), ctx);
+ return EnsureOptionalType(node.Pos(), *node.GetTypeAnn(), ctx);
}
bool EnsureOptionalType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
@@ -2558,13 +2558,13 @@ bool EnsureOptionalType(TPositionHandle position, const TTypeAnnotationNode& typ
bool EnsureType(const TExprNode& node, TExprContext& ctx) {
YQL_ENSURE(!node.IsCallable({"SqlColumnOrType", "SqlPlainColumnOrType", "SqlColumnFromType"}),
"Unexpected " << node.Content() << " it should be processed earlier");
- if (!node.GetTypeAnn()) {
+ if (!node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Type) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Type) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2628,13 +2628,13 @@ bool EnsureDryType(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureDictType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected dict type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Dict) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Dict) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected dict type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2652,13 +2652,13 @@ bool EnsureDictType(TPositionHandle position, const TTypeAnnotationNode& type, T
}
bool EnsureVoidType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Void) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Void) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2667,7 +2667,7 @@ bool EnsureVoidType(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureVoidLiteral(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected void literal, but got lambda"));
return false;
@@ -2682,13 +2682,13 @@ bool EnsureVoidLiteral(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureCallableType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected callable type, but got lambda"));
return false;
}
- return EnsureCallableType(node.Pos(), *node.GetTypeAnn(), ctx);
+ return EnsureCallableType(node.Pos(), *node.GetTypeAnn(), ctx);
}
bool EnsureCallableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
@@ -2701,13 +2701,13 @@ bool EnsureCallableType(TPositionHandle position, const TTypeAnnotationNode& typ
}
bool EnsureResourceType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected resource type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Resource) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Resource) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected resource type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2716,13 +2716,13 @@ bool EnsureResourceType(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureTaggedType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tagged type, but got lambda"));
return false;
}
- if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tagged) {
+ if (node.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Tagged) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected tagged type, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -2731,14 +2731,14 @@ bool EnsureTaggedType(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureOneOrTupleOfDataOrOptionalOfData(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
+ if (HasError(node.GetTypeAnn(), ctx) || !node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() <<
"Expected either data (optional of data) or non-empty tuple of data (optional of data), but got lambda"));
return false;
}
- return EnsureOneOrTupleOfDataOrOptionalOfData(node.Pos(), *node.GetTypeAnn(), ctx);
+ return EnsureOneOrTupleOfDataOrOptionalOfData(node.Pos(), *node.GetTypeAnn(), ctx);
}
bool EnsureOneOrTupleOfDataOrOptionalOfData(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
@@ -2775,12 +2775,12 @@ bool EnsureComparableType(TPositionHandle position, const TTypeAnnotationNode& t
return false;
}
- if (!type.IsComparable()) {
+ if (!type.IsComparable()) {
ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder()
<< "Expected comparable type, i.e. combination of Data, Optional, List or Tuple, but got:" << type));
return false;
}
- return true;
+ return true;
}
bool EnsureEquatableType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
@@ -2788,12 +2788,12 @@ bool EnsureEquatableType(TPositionHandle position, const TTypeAnnotationNode& ty
return false;
}
- if (!type.IsEquatable()) {
+ if (!type.IsEquatable()) {
ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder()
- << "Expected equatable type, i.e. combination of Data, Optional, List, Dict, Tuple, Struct, or Variant, but got:" << type));
+ << "Expected equatable type, i.e. combination of Data, Optional, List, Dict, Tuple, Struct, or Variant, but got:" << type));
return false;
}
- return true;
+ return true;
}
bool IsInstantEqual(const TTypeAnnotationNode& type) {
@@ -2829,7 +2829,7 @@ bool IsInstantEqual(const TTypeAnnotationNode& type) {
static bool EnsureKeyProperty(TPositionHandle position, const TTypeAnnotationNode* keyType,
const std::function<bool(TPositionHandle, EDataSlot, TExprContext&)>& propertyCheck, TExprContext& ctx) {
-
+
if (HasError(keyType, ctx)) {
return false;
}
@@ -2861,17 +2861,17 @@ static bool EnsureKeyProperty(TPositionHandle position, const TTypeAnnotationNod
}
bool EnsureHashableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx) {
- return EnsureKeyProperty(position, keyType, EnsureHashableDataType, ctx);
-}
-
+ return EnsureKeyProperty(position, keyType, EnsureHashableDataType, ctx);
+}
+
bool EnsureComparableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx) {
- return EnsureKeyProperty(position, keyType, EnsureComparableDataType, ctx);
-}
-
+ return EnsureKeyProperty(position, keyType, EnsureComparableDataType, ctx);
+}
+
bool EnsureEquatableKey(TPositionHandle position, const TTypeAnnotationNode* keyType, TExprContext& ctx) {
- return EnsureKeyProperty(position, keyType, EnsureEquatableDataType, ctx);
-}
-
+ return EnsureKeyProperty(position, keyType, EnsureEquatableDataType, ctx);
+}
+
bool UpdateLambdaAllArgumentsTypes(TExprNode::TPtr& lambda, const std::vector<const TTypeAnnotationNode*>& argumentsAnnotations, TExprContext& ctx) {
YQL_ENSURE(lambda->Type() == TExprNode::Lambda);
@@ -2879,31 +2879,31 @@ bool UpdateLambdaAllArgumentsTypes(TExprNode::TPtr& lambda, const std::vector<co
auto argsChildren = args.ChildrenList();
YQL_ENSURE(argsChildren.size() == argumentsAnnotations.size());
- bool updateArgs = false;
+ bool updateArgs = false;
for (size_t i = 0U; i < argumentsAnnotations.size(); ++i) {
const auto arg = args.Child(i);
- if (!arg->GetTypeAnn() || !IsSameAnnotation(*arg->GetTypeAnn(), *argumentsAnnotations[i])) {
- updateArgs = true;
- break;
+ if (!arg->GetTypeAnn() || !IsSameAnnotation(*arg->GetTypeAnn(), *argumentsAnnotations[i])) {
+ updateArgs = true;
+ break;
}
}
- if (!updateArgs && args.GetTypeAnn()) {
+ if (!updateArgs && args.GetTypeAnn()) {
return true;
}
- TNodeOnNodeOwnedMap replaces;
- replaces.reserve(argumentsAnnotations.size());
-
- for (size_t i = 0U; i < argumentsAnnotations.size(); ++i) {
- const auto arg = args.Child(i);
- auto newArg = ctx.ShallowCopy(*arg);
- newArg->SetTypeAnn(argumentsAnnotations[i]);
- YQL_ENSURE(replaces.emplace(arg, newArg).second);
- argsChildren[i] = std::move(newArg);
- }
-
+ TNodeOnNodeOwnedMap replaces;
+ replaces.reserve(argumentsAnnotations.size());
+
+ for (size_t i = 0U; i < argumentsAnnotations.size(); ++i) {
+ const auto arg = args.Child(i);
+ auto newArg = ctx.ShallowCopy(*arg);
+ newArg->SetTypeAnn(argumentsAnnotations[i]);
+ YQL_ENSURE(replaces.emplace(arg, newArg).second);
+ argsChildren[i] = std::move(newArg);
+ }
+
auto newArgs = ctx.NewArguments(args.Pos(), std::move(argsChildren));
- newArgs->SetTypeAnn(ctx.MakeType<TUnitExprType>());
+ newArgs->SetTypeAnn(ctx.MakeType<TUnitExprType>());
lambda = ctx.NewLambda(lambda->Pos(), std::move(newArgs), ctx.ReplaceNodes(GetLambdaBody(*lambda), replaces));
return true;
}
@@ -2912,12 +2912,12 @@ bool UpdateLambdaArgumentsType(const TExprNode& lambda, TExprContext& ctx) {
YQL_ENSURE(lambda.Type() == TExprNode::Lambda);
const auto args = lambda.Child(0);
YQL_ENSURE(0U == args->ChildrenSize());
- args->SetTypeAnn(ctx.MakeType<TUnitExprType>());
+ args->SetTypeAnn(ctx.MakeType<TUnitExprType>());
return true;
}
bool EnsureDataOrOptionalOfData(const TExprNode& node, bool& isOptional, const TDataExprType*& dataType, TExprContext& ctx) {
- if (!node.GetTypeAnn()) {
+ if (!node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
}
@@ -2942,9 +2942,9 @@ bool EnsurePersistable(const TExprNode& node, TExprContext& ctx) {
}
if (!node.IsPersistable()) {
- if (node.GetTypeAnn()) {
+ if (node.GetTypeAnn()) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() <<
- "Expected persistable data, but got: " << *node.GetTypeAnn()));
+ "Expected persistable data, but got: " << *node.GetTypeAnn()));
} else {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() <<
"Expected persistable data, but got lambda"));
@@ -2975,9 +2975,9 @@ bool EnsurePersistableType(TPositionHandle position, const TTypeAnnotationNode&
bool EnsureComputable(const TExprNode& node, TExprContext& ctx) {
if (!node.IsComputable()) {
- if (node.GetTypeAnn()) {
+ if (node.GetTypeAnn()) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() <<
- "Expected computable data, but got: " << *node.GetTypeAnn()));
+ "Expected computable data, but got: " << *node.GetTypeAnn()));
}
else {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() <<
@@ -2993,9 +2993,9 @@ bool EnsureComputable(const TExprNode& node, TExprContext& ctx) {
bool EnsureInspectable(const TExprNode& node, TExprContext& ctx) {
if (!node.IsInspectable()) {
- if (node.GetTypeAnn()) {
+ if (node.GetTypeAnn()) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() <<
- "Expected inspectable data, but got: " << *node.GetTypeAnn()));
+ "Expected inspectable data, but got: " << *node.GetTypeAnn()));
}
else {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() <<
@@ -3034,17 +3034,17 @@ bool EnsureComputableType(TPositionHandle position, const TTypeAnnotationNode& t
}
bool EnsureListOrOptionalType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx)) {
+ if (HasError(node.GetTypeAnn(), ctx)) {
return false;
}
- if (!node.GetTypeAnn()) {
+ if (!node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional, but got lambda"));
return false;
}
- auto kind = node.GetTypeAnn()->GetKind();
+ auto kind = node.GetTypeAnn()->GetKind();
if (kind != ETypeAnnotationKind::List && kind != ETypeAnnotationKind::Optional) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or optional, but got: " << *node.GetTypeAnn()));
return false;
@@ -3083,17 +3083,17 @@ bool EnsureListOrOptionalListType(const TExprNode& node, TExprContext& ctx) {
}
bool EnsureSeqType(const TExprNode& node, TExprContext& ctx, bool* isStream) {
- if (HasError(node.GetTypeAnn(), ctx)) {
+ if (HasError(node.GetTypeAnn(), ctx)) {
return false;
}
- if (!node.GetTypeAnn()) {
+ if (!node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list or stream, but got lambda"));
return false;
}
- return EnsureSeqType(node.Pos(), *node.GetTypeAnn(), ctx, isStream);
+ return EnsureSeqType(node.Pos(), *node.GetTypeAnn(), ctx, isStream);
}
bool EnsureSeqType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx, bool* isStream) {
@@ -3117,17 +3117,17 @@ bool EnsureSeqType(TPositionHandle position, const TTypeAnnotationNode& type, TE
}
bool EnsureSeqOrOptionalType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx)) {
+ if (HasError(node.GetTypeAnn(), ctx)) {
return false;
}
- if (!node.GetTypeAnn()) {
+ if (!node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either list, stream or optional, but got lambda"));
return false;
}
- auto kind = node.GetTypeAnn()->GetKind();
+ auto kind = node.GetTypeAnn()->GetKind();
if (kind != ETypeAnnotationKind::List && kind != ETypeAnnotationKind::Stream && kind != ETypeAnnotationKind::Optional) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected list, stream or optional, but got: " << *node.GetTypeAnn()));
return false;
@@ -3221,17 +3221,17 @@ template bool EnsureNewSeqType<false, true, true>(const TExprNode& node, TExprCo
template bool EnsureNewSeqType<false, true, false>(const TExprNode& node, TExprContext& ctx, const TTypeAnnotationNode** itemType);
bool EnsureStructOrOptionalStructType(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx)) {
+ if (HasError(node.GetTypeAnn(), ctx)) {
return false;
}
- if (!node.GetTypeAnn()) {
+ if (!node.GetTypeAnn()) {
YQL_ENSURE(node.Type() == TExprNode::Lambda);
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected either struct or optional of struct, but got lambda"));
return false;
}
- return EnsureStructOrOptionalStructType(node.Pos(), *node.GetTypeAnn(), ctx);
+ return EnsureStructOrOptionalStructType(node.Pos(), *node.GetTypeAnn(), ctx);
}
bool EnsureStructOrOptionalStructType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
@@ -3248,7 +3248,7 @@ bool EnsureStructOrOptionalStructType(TPositionHandle position, const TTypeAnnot
auto itemType = type.Cast<TOptionalExprType>()->GetItemType();
kind = itemType->GetKind();
if (kind != ETypeAnnotationKind::Struct) {
- ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected either struct or optional of struct, but got: " << type));
+ ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected either struct or optional of struct, but got: " << type));
return false;
}
}
@@ -3257,7 +3257,7 @@ bool EnsureStructOrOptionalStructType(TPositionHandle position, const TTypeAnnot
}
bool EnsureDependsOn(const TExprNode& node, TExprContext& ctx) {
- if (HasError(node.GetTypeAnn(), ctx)) {
+ if (HasError(node.GetTypeAnn(), ctx)) {
return false;
}
@@ -3295,7 +3295,7 @@ bool EnsureTypeHandleResourceType(const TExprNode& node, TExprContext& ctx) {
return false;
}
- if (node.GetTypeAnn()->Cast<TResourceExprType>()->GetTag() != TypeResourceTag) {
+ if (node.GetTypeAnn()->Cast<TResourceExprType>()->GetTag() != TypeResourceTag) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type handle, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -3312,7 +3312,7 @@ bool EnsureCodeResourceType(const TExprNode& node, TExprContext& ctx) {
return false;
}
- if (node.GetTypeAnn()->Cast<TResourceExprType>()->GetTag() != CodeResourceTag) {
+ if (node.GetTypeAnn()->Cast<TResourceExprType>()->GetTag() != CodeResourceTag) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Expected type handle, but got: " << *node.GetTypeAnn()));
return false;
}
@@ -3334,7 +3334,7 @@ const TTypeAnnotationNode* MakeSequenceType(ETypeAnnotationKind sequenceKind, co
IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& expectedType,
TExprContext& ctx, TConvertFlags flags) {
- if (HasError(node->GetTypeAnn(), ctx)) {
+ if (HasError(node->GetTypeAnn(), ctx)) {
return IGraphTransformer::TStatus::Error;
}
@@ -3362,7 +3362,7 @@ IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnota
IGraphTransformer::TStatus TryConvertTo(TExprNode::TPtr& node, const TTypeAnnotationNode& sourceType,
const TTypeAnnotationNode& expectedType, TExprContext& ctx, TConvertFlags flags) {
- if (HasError(node->GetTypeAnn(), ctx)) {
+ if (HasError(node->GetTypeAnn(), ctx)) {
return IGraphTransformer::TStatus::Error;
}
@@ -3531,7 +3531,7 @@ TMaybe<EDataSlot> GetSuperType(EDataSlot dataSlot1, EDataSlot dataSlot2) {
IGraphTransformer::TStatus SilentInferCommonType(TExprNode::TPtr& node1, TExprNode::TPtr& node2, TExprContext& ctx,
const TTypeAnnotationNode*& commonType, TConvertFlags flags) {
- if (!node1->GetTypeAnn() || !node2->GetTypeAnn()) {
+ if (!node1->GetTypeAnn() || !node2->GetTypeAnn()) {
return IGraphTransformer::TStatus::Error;
}
@@ -4172,7 +4172,7 @@ bool IsPureIsolatedLambdaImpl(const TExprNode& lambdaBody, TNodeSet& visited) {
return true;
}
- if (!lambdaBody.GetTypeAnn()->IsComposable()) {
+ if (!lambdaBody.GetTypeAnn()->IsComposable()) {
return false;
}
@@ -4321,20 +4321,20 @@ void ExtractIntegralValue(const TExprNode& constructor, bool negate, bool& hasSi
}
TMaybe<ui32> GetDataFixedSize(const TTypeAnnotationNode* typeAnnotation) {
- if (!typeAnnotation) {
+ if (!typeAnnotation) {
return Nothing();
- }
-
- if (typeAnnotation->GetKind() == ETypeAnnotationKind::Optional) {
+ }
+
+ if (typeAnnotation->GetKind() == ETypeAnnotationKind::Optional) {
auto childSize = GetDataFixedSize(typeAnnotation->Cast<TOptionalExprType>()->GetItemType());
if (childSize) {
return 1 + *childSize;
}
return Nothing();
- }
-
- if (typeAnnotation->GetKind() == ETypeAnnotationKind::Data) {
+ }
+
+ if (typeAnnotation->GetKind() == ETypeAnnotationKind::Data) {
const auto dataSlot = typeAnnotation->Cast<TDataExprType>()->GetSlot();
if (EDataSlot::Bool == dataSlot || EDataSlot::Uint8 == dataSlot || EDataSlot::Int8 == dataSlot) {
return 1;
@@ -4355,21 +4355,21 @@ TMaybe<ui32> GetDataFixedSize(const TTypeAnnotationNode* typeAnnotation) {
}
return Nothing();
- }
-
- if (typeAnnotation->GetKind() == ETypeAnnotationKind::Tuple) {
+ }
+
+ if (typeAnnotation->GetKind() == ETypeAnnotationKind::Tuple) {
ui32 totalSize = 0;
for (auto& child : typeAnnotation->Cast<TTupleExprType>()->GetItems()) {
auto childSize = GetDataFixedSize(child);
if (!childSize) {
return Nothing();
- }
+ }
totalSize += *childSize;
- }
+ }
return totalSize;
- }
+ }
if (typeAnnotation->GetKind() == ETypeAnnotationKind::Struct) {
ui32 totalSize = 0;
@@ -4417,14 +4417,14 @@ TMaybe<ui32> GetDataFixedSize(const TTypeAnnotationNode* typeAnnotation) {
return Nothing();
}
-
+
bool IsFixedSizeData(const TTypeAnnotationNode* typeAnnotation) {
return GetDataFixedSize(typeAnnotation).Defined();
}
bool IsDataTypeString(EDataSlot dataSlot) {
return NUdf::GetDataTypeInfo(dataSlot).Features & NUdf::StringType;
-}
+}
bool EnsureComparableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx) {
if (0 == (NUdf::GetDataTypeInfo(dataSlot).Features & NUdf::CanCompare)) {
@@ -4450,21 +4450,21 @@ bool EnsureComparableDataType(TPositionHandle position, EDataSlot dataSlot, TExp
bool EnsureEquatableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx) {
if (0 == (NUdf::GetDataTypeInfo(dataSlot).Features & NUdf::CanEquate)) {
ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected equatable type, but got: " << NKikimr::NUdf::GetDataTypeInfo(dataSlot).Name));
- return false;
- }
-
- return true;
-}
-
+ return false;
+ }
+
+ return true;
+}
+
bool EnsureHashableDataType(TPositionHandle position, EDataSlot dataSlot, TExprContext& ctx) {
if (0 == (NUdf::GetDataTypeInfo(dataSlot).Features & NUdf::CanHash)) {
ctx.AddError(TIssue(ctx.GetPosition(position), TStringBuilder() << "Expected hashable type, but got: " << NKikimr::NUdf::GetDataTypeInfo(dataSlot).Name));
- return false;
- }
-
- return true;
-}
-
+ return false;
+ }
+
+ return true;
+}
+
TMaybe<TIssue> NormalizeName(TPosition position, TString& name) {
const ui32 inputLength = name.length();
ui32 startCharPos = 0;
@@ -4548,7 +4548,7 @@ bool HasError(const TTypeAnnotationNode* type, TIssue& errIssue) {
}
bool IsNull(const TExprNode& node) {
- return node.GetTypeAnn() && IsNull(*node.GetTypeAnn());
+ return node.GetTypeAnn() && IsNull(*node.GetTypeAnn());
}
bool IsNull(const TTypeAnnotationNode& type) {
@@ -4563,67 +4563,67 @@ bool IsEmptyList(const TTypeAnnotationNode& type) {
return type.GetKind() == ETypeAnnotationKind::EmptyList;
}
-static TString GetStructDiff(const TStructExprType& left, const TStructExprType& right) {
+static TString GetStructDiff(const TStructExprType& left, const TStructExprType& right) {
THashMap<TStringBuf, const TItemExprType*> rightItems;
- for (auto item: right.GetItems()) {
- rightItems.insert({item->GetName(), item});
- }
- TStringBuilder res;
- for (auto item: left.GetItems()) {
- if (auto rightItem = rightItems.Value(item->GetName(), nullptr)) {
- if (!IsSameAnnotation(*item, *rightItem)) {
+ for (auto item: right.GetItems()) {
+ rightItems.insert({item->GetName(), item});
+ }
+ TStringBuilder res;
+ for (auto item: left.GetItems()) {
+ if (auto rightItem = rightItems.Value(item->GetName(), nullptr)) {
+ if (!IsSameAnnotation(*item, *rightItem)) {
res << item->GetName() << '(' << GetTypeDiff(*item->GetItemType(), *rightItem->GetItemType()) << TStringBuf("),");
- }
- rightItems.erase(item->GetName());
- } else {
+ }
+ rightItems.erase(item->GetName());
+ } else {
res << '-' << item->GetName() << '(' << *item->GetItemType() << TStringBuf("),");
- }
- }
- for (auto& item: rightItems) {
+ }
+ }
+ for (auto& item: rightItems) {
res << '+' << item.first << '(' << *item.second->GetItemType() << TStringBuf("),");
- }
+ }
if (!res.empty()) {
- return res.pop_back(); // remove trailing comma
- }
- return "no diff";
-}
-
-TString GetTypeDiff(const TTypeAnnotationNode& left, const TTypeAnnotationNode& right) {
- if (&left == &right) {
- return "no diff";
- }
- TStringBuilder res;
- if (left.GetKind() == right.GetKind()) {
- switch (left.GetKind()) {
- case ETypeAnnotationKind::List:
+ return res.pop_back(); // remove trailing comma
+ }
+ return "no diff";
+}
+
+TString GetTypeDiff(const TTypeAnnotationNode& left, const TTypeAnnotationNode& right) {
+ if (&left == &right) {
+ return "no diff";
+ }
+ TStringBuilder res;
+ if (left.GetKind() == right.GetKind()) {
+ switch (left.GetKind()) {
+ case ETypeAnnotationKind::List:
res << TStringBuf("List<")
- << GetTypeDiff(*left.Cast<TListExprType>()->GetItemType(), *right.Cast<TListExprType>()->GetItemType())
- << '>';
- return res;
- case ETypeAnnotationKind::Stream:
+ << GetTypeDiff(*left.Cast<TListExprType>()->GetItemType(), *right.Cast<TListExprType>()->GetItemType())
+ << '>';
+ return res;
+ case ETypeAnnotationKind::Stream:
res << TStringBuf("Stream<")
- << GetTypeDiff(*left.Cast<TStreamExprType>()->GetItemType(), *right.Cast<TStreamExprType>()->GetItemType())
- << '>';
- return res;
- case ETypeAnnotationKind::Struct:
+ << GetTypeDiff(*left.Cast<TStreamExprType>()->GetItemType(), *right.Cast<TStreamExprType>()->GetItemType())
+ << '>';
+ return res;
+ case ETypeAnnotationKind::Struct:
res << TStringBuf("Struct<")
- << GetStructDiff(*left.Cast<TStructExprType>(), *right.Cast<TStructExprType>())
- << '>';
- return res;
+ << GetStructDiff(*left.Cast<TStructExprType>(), *right.Cast<TStructExprType>())
+ << '>';
+ return res;
case ETypeAnnotationKind::Flow:
res << TStringBuf("Flow<")
<< GetTypeDiff(*left.Cast<TFlowExprType>()->GetItemType(), *right.Cast<TFlowExprType>()->GetItemType())
<< '>';
return res;
- default:
+ default:
res << left << TStringBuf("!=") << right;
- return res;
- }
- }
+ return res;
+ }
+ }
res << left.GetKind() << TStringBuf("!=") << right.GetKind();
- return res;
-}
-
+ return res;
+}
+
TExprNode::TPtr ExpandTypeNoCache(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx) {
switch (type.GetKind()) {
case ETypeAnnotationKind::Unit:
@@ -4914,25 +4914,25 @@ std::optional<ui32> GetFieldPosition(const TStructExprType& structType, const TS
return std::nullopt;
}
-bool IsCallableTypeHasStreams(const TCallableExprType* callableType) {
- for (;;) {
- if (callableType->GetReturnType()->GetKind() == ETypeAnnotationKind::Stream) {
- return true;
- }
- else {
- for (auto& arg: callableType->GetArguments()) {
- if (arg.Type->GetKind() == ETypeAnnotationKind::Stream) {
- return true;
- }
- }
- }
- if (callableType->GetReturnType()->GetKind() == ETypeAnnotationKind::Callable) {
- callableType = callableType->GetReturnType()->Cast<TCallableExprType>();
- } else {
- break;
- }
- }
- return false;
-}
-
-} // NYql
+bool IsCallableTypeHasStreams(const TCallableExprType* callableType) {
+ for (;;) {
+ if (callableType->GetReturnType()->GetKind() == ETypeAnnotationKind::Stream) {
+ return true;
+ }
+ else {
+ for (auto& arg: callableType->GetArguments()) {
+ if (arg.Type->GetKind() == ETypeAnnotationKind::Stream) {
+ return true;
+ }
+ }
+ }
+ if (callableType->GetReturnType()->GetKind() == ETypeAnnotationKind::Callable) {
+ callableType = callableType->GetReturnType()->Cast<TCallableExprType>();
+ } else {
+ break;
+ }
+ }
+ return false;
+}
+
+} // NYql
diff --git a/ydb/library/yql/core/yql_expr_type_annotation.h b/ydb/library/yql/core/yql_expr_type_annotation.h
index a865aa27cd..8f7b52d627 100644
--- a/ydb/library/yql/core/yql_expr_type_annotation.h
+++ b/ydb/library/yql/core/yql_expr_type_annotation.h
@@ -264,9 +264,9 @@ bool IsEmptyList(const TExprNode& node);
bool IsEmptyList(const TTypeAnnotationNode& type);
bool IsInstantEqual(const TTypeAnnotationNode& type);
-TString GetTypeDiff(const TTypeAnnotationNode& left, const TTypeAnnotationNode& right);
+TString GetTypeDiff(const TTypeAnnotationNode& left, const TTypeAnnotationNode& right);
TExprNode::TPtr ExpandType(TPositionHandle position, const TTypeAnnotationNode& type, TExprContext& ctx);
-
+
bool IsSystemMember(const TStringBuf& memberName);
IGraphTransformer::TStatus NormalizeTupleOfAtoms(const TExprNode::TPtr& input, ui32 index, TExprNode::TPtr& output, TExprContext& ctx,
@@ -278,6 +278,6 @@ std::optional<ui32> GetFieldPosition(const TMultiExprType& tupleType, const TStr
std::optional<ui32> GetFieldPosition(const TTupleExprType& tupleType, const TStringBuf& field);
std::optional<ui32> GetFieldPosition(const TStructExprType& structType, const TStringBuf& field);
-bool IsCallableTypeHasStreams(const TCallableExprType* callableType);
-
+bool IsCallableTypeHasStreams(const TCallableExprType* callableType);
+
}
diff --git a/ydb/library/yql/core/yql_graph_transformer.h b/ydb/library/yql/core/yql_graph_transformer.h
index 60d87727dd..52cdaed299 100644
--- a/ydb/library/yql/core/yql_graph_transformer.h
+++ b/ydb/library/yql/core/yql_graph_transformer.h
@@ -4,12 +4,12 @@
#include <ydb/library/yql/core/issue/yql_issue.h>
#include <library/cpp/threading/future/future.h>
-
+
#include <util/generic/hash.h>
-#include <util/datetime/base.h>
-
-#include <functional>
+#include <util/datetime/base.h>
+#include <functional>
+
namespace NYql {
class IGraphTransformer {
@@ -75,7 +75,7 @@ public:
TDuration WaitDuration;
i32 NewExprNodes;
i32 NewTypeNodes;
- i32 NewConstraintNodes;
+ i32 NewConstraintNodes;
ui32 Repeats;
ui32 Restarts;
@@ -86,7 +86,7 @@ public:
, WaitDuration(TDuration::Zero())
, NewExprNodes(0)
, NewTypeNodes(0)
- , NewConstraintNodes(0)
+ , NewConstraintNodes(0)
, Repeats(0)
, Restarts(0)
, Stages() {}
@@ -114,17 +114,17 @@ private:
, ExprCtx(exprCtx)
, TransformStart(TInstant::Now())
, ExprNodesSize(exprCtx ? exprCtx->ExprNodes.size() : 0)
- , TypeNodesSize(exprCtx ? exprCtx->TypeNodes.size() : 0)
- , ConstraintNodesSize(exprCtx ? exprCtx->ConstraintNodes.size() : 0)
- {
- }
+ , TypeNodesSize(exprCtx ? exprCtx->TypeNodes.size() : 0)
+ , ConstraintNodesSize(exprCtx ? exprCtx->ConstraintNodes.size() : 0)
+ {
+ }
~TTransformScope() {
Statistics.TransformDuration += TInstant::Now() - TransformStart;
if (ExprCtx) {
Statistics.NewExprNodes += ExprCtx->ExprNodes.size() - ExprNodesSize;
Statistics.NewTypeNodes += ExprCtx->TypeNodes.size() - TypeNodesSize;
- Statistics.NewConstraintNodes += ExprCtx->ConstraintNodes.size() - ConstraintNodesSize;
+ Statistics.NewConstraintNodes += ExprCtx->ConstraintNodes.size() - ConstraintNodesSize;
}
}
@@ -146,7 +146,7 @@ private:
TInstant TransformStart;
i64 ExprNodesSize;
i64 TypeNodesSize;
- i64 ConstraintNodesSize;
+ i64 ConstraintNodesSize;
};
public:
@@ -312,15 +312,15 @@ public:
return status;
}
- void Rewind() override {
- Callbacks.clear();
- }
-
+ void Rewind() override {
+ Callbacks.clear();
+ }
+
private:
TNodeMap<TAsyncTransformCallbackFuture> Callbacks;
};
-template <typename TFuture, typename TCallback>
+template <typename TFuture, typename TCallback>
std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture>
WrapFutureCallback(const TFuture& future, const TCallback& callback, const TString& message = "") {
return std::make_pair(IGraphTransformer::TStatus::Async, future.Apply(
@@ -351,30 +351,30 @@ WrapFutureCallback(const TFuture& future, const TCallback& callback, const TStri
}));
}
-template <typename TFuture, typename TResultExtractor>
-std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture>
+template <typename TFuture, typename TResultExtractor>
+std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture>
WrapFuture(const TFuture& future, const TResultExtractor& extractor, const TString& message = "") {
- return WrapFutureCallback(future, [extractor](const NThreading::TFutureType<TFuture>& res, const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
+ return WrapFutureCallback(future, [extractor](const NThreading::TFutureType<TFuture>& res, const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
input->SetState(TExprNode::EState::ExecutionComplete);
input->SetResult(extractor(res, input, ctx));
- return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Ok);
+ return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Ok);
}, message);
-}
-
-template <typename TFuture, typename TResultExtractor>
-std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture>
+}
+
+template <typename TFuture, typename TResultExtractor>
+std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture>
WrapModifyFuture(const TFuture& future, const TResultExtractor& extractor, const TString& message = "") {
- return WrapFutureCallback(future, [extractor](const NThreading::TFutureType<TFuture>& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- TExprNode::TPtr resultNode = extractor(res, input, output, ctx);
+ return WrapFutureCallback(future, [extractor](const NThreading::TFutureType<TFuture>& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ TExprNode::TPtr resultNode = extractor(res, input, output, ctx);
input->SetState(TExprNode::EState::ExecutionComplete);
- output->SetResult(std::move(resultNode));
- if (input != output) {
- return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true);
- }
- return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Ok);
+ output->SetResult(std::move(resultNode));
+ if (input != output) {
+ return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat, true);
+ }
+ return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Ok);
}, message);
-}
-
+}
+
inline std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> SyncStatus(IGraphTransformer::TStatus status) {
return std::make_pair(status, TAsyncTransformCallbackFuture());
}
diff --git a/ydb/library/yql/core/yql_join.cpp b/ydb/library/yql/core/yql_join.cpp
index 9a9ded7a32..a07414e045 100644
--- a/ydb/library/yql/core/yql_join.cpp
+++ b/ydb/library/yql/core/yql_join.cpp
@@ -400,7 +400,7 @@ namespace {
TMaybe<TIssue> TJoinLabel::Parse(TExprContext& ctx, TExprNode& node, const TStructExprType* structType) {
Tables.clear();
InputType = structType;
- if (auto atom = TMaybeNode<TCoAtom>(&node)) {
+ if (auto atom = TMaybeNode<TCoAtom>(&node)) {
if (auto err = ValidateLabel(ctx, atom.Cast())) {
return err;
}
@@ -409,7 +409,7 @@ TMaybe<TIssue> TJoinLabel::Parse(TExprContext& ctx, TExprNode& node, const TStru
Tables.push_back(atom.Cast().Value());
return {};
}
- else if (auto tuple = TMaybeNode<TCoAtomList>(&node)) {
+ else if (auto tuple = TMaybeNode<TCoAtomList>(&node)) {
if (tuple.Cast().Size() == 0) {
return TIssue(ctx.GetPosition(node.Pos()), "Empty list of correlation names are not allowed");
}
@@ -620,14 +620,14 @@ TVector<TString> TJoinLabels::EnumerateColumns(const TStringBuf& table) const {
IGraphTransformer::TStatus ValidateEquiJoinOptions(TPositionHandle positionHandle, const TExprNode& optionsNode,
TJoinOptions& options, TExprContext& ctx)
-{
+{
auto position = ctx.GetPosition(positionHandle);
if (!EnsureTuple(optionsNode, ctx)) {
return IGraphTransformer::TStatus::Error;
}
options = TJoinOptions{};
-
+
THashSet<TStringBuf> renameTargetSet;
bool hasRename = false;
for (auto child : optionsNode.Children()) {
@@ -666,7 +666,7 @@ IGraphTransformer::TStatus ValidateEquiJoinOptions(TPositionHandle positionHandl
}
} else if (optionName == "flatten") {
options.Flatten = true;
- } else if (optionName == "keep_sys") {
+ } else if (optionName == "keep_sys") {
options.KeepSysColumns = true;
} else if (optionName == "strict_keys") {
options.StrictKeys = true;
@@ -707,19 +707,19 @@ IGraphTransformer::TStatus ValidateEquiJoinOptions(TPositionHandle positionHandl
}
}
- return IGraphTransformer::TStatus::Ok;
-}
-
-IGraphTransformer::TStatus EquiJoinAnnotation(
- TPositionHandle positionHandle,
- const TStructExprType*& resultType,
- const TJoinLabels& labels,
- const TExprNode& joins,
+ return IGraphTransformer::TStatus::Ok;
+}
+
+IGraphTransformer::TStatus EquiJoinAnnotation(
+ TPositionHandle positionHandle,
+ const TStructExprType*& resultType,
+ const TJoinLabels& labels,
+ const TExprNode& joins,
const TJoinOptions& options,
- TExprContext& ctx
-) {
- auto position = ctx.GetPosition(positionHandle);
-
+ TExprContext& ctx
+) {
+ auto position = ctx.GetPosition(positionHandle);
+
if (labels.InputByTable.size() < 2) {
ctx.AddError(TIssue(position, TStringBuilder() << "Expected at least 2 table"));
return IGraphTransformer::TStatus::Error;
@@ -750,7 +750,7 @@ IGraphTransformer::TStatus EquiJoinAnnotation(
auto columnTypes = GetJoinColumnTypes(joins, labels, ctx);
TVector<const TItemExprType*> resultFields;
TMap<TString, TFlattenState> flattenFields; // column -> table
- THashSet<TString> processedRenames;
+ THashSet<TString> processedRenames;
for (auto it: labels.Inputs) {
for (auto item: it.InputType->GetItems()) {
TString fullName = it.FullName(item->GetName());
@@ -758,12 +758,12 @@ IGraphTransformer::TStatus EquiJoinAnnotation(
if (type) {
TVector<TStringBuf> fullNames;
fullNames.push_back(fullName);
- if (!processedRenames.contains(fullName)) {
+ if (!processedRenames.contains(fullName)) {
auto renameIt = options.RenameMap.find(fullName);
if (renameIt != options.RenameMap.end()) {
- fullNames = renameIt->second;
- processedRenames.insert(fullName);
- }
+ fullNames = renameIt->second;
+ processedRenames.insert(fullName);
+ }
}
for (auto& fullName: fullNames) {
@@ -1006,7 +1006,7 @@ std::pair<bool, bool> IsRequiredSide(const TExprNode::TPtr& joinTree, const TJoi
auto joinType = joinTree->Child(0)->Content();
auto left = joinTree->ChildPtr(1);
auto right = joinTree->ChildPtr(2);
- if (joinType == "Inner" || joinType == "Left" || joinType == "LeftOnly" || joinType == "LeftSemi" || joinType == "RightSemi" || joinType == "Cross") {
+ if (joinType == "Inner" || joinType == "Left" || joinType == "LeftOnly" || joinType == "LeftSemi" || joinType == "RightSemi" || joinType == "Cross") {
if (!left->IsAtom()) {
auto x = IsRequiredSide(left, labels, inputIndex);
if (x.first) {
@@ -1021,7 +1021,7 @@ std::pair<bool, bool> IsRequiredSide(const TExprNode::TPtr& joinTree, const TJoi
}
}
- if (joinType == "Inner" || joinType == "Right" || joinType == "RightOnly" || joinType == "RightSemi" || joinType == "LeftSemi" || joinType == "Cross") {
+ if (joinType == "Inner" || joinType == "Right" || joinType == "RightOnly" || joinType == "RightSemi" || joinType == "LeftSemi" || joinType == "Cross") {
if (!right->IsAtom()) {
auto x = IsRequiredSide(right, labels, inputIndex);
if (x.first) {
diff --git a/ydb/library/yql/core/yql_join.h b/ydb/library/yql/core/yql_join.h
index b01b950c2d..fb99788985 100644
--- a/ydb/library/yql/core/yql_join.h
+++ b/ydb/library/yql/core/yql_join.h
@@ -2,11 +2,11 @@
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/core/yql_graph_transformer.h>
-
+
#include <util/generic/set.h>
-#include <util/generic/vector.h>
-#include <util/generic/hash.h>
-#include <util/generic/strbuf.h>
+#include <util/generic/vector.h>
+#include <util/generic/hash.h>
+#include <util/generic/strbuf.h>
namespace NYql {
@@ -59,13 +59,13 @@ struct TJoinOptions {
bool StrictKeys = false;
};
-IGraphTransformer::TStatus ValidateEquiJoinOptions(
- TPositionHandle positionHandle,
+IGraphTransformer::TStatus ValidateEquiJoinOptions(
+ TPositionHandle positionHandle,
const TExprNode& optionsNode,
TJoinOptions& options,
- TExprContext& ctx
-);
-
+ TExprContext& ctx
+);
+
IGraphTransformer::TStatus EquiJoinAnnotation(
TPositionHandle position,
const TStructExprType*& resultType,
diff --git a/ydb/library/yql/core/yql_opt_proposed_by_data.cpp b/ydb/library/yql/core/yql_opt_proposed_by_data.cpp
index 23a912303f..2d1a58bd56 100644
--- a/ydb/library/yql/core/yql_opt_proposed_by_data.cpp
+++ b/ydb/library/yql/core/yql_opt_proposed_by_data.cpp
@@ -25,7 +25,7 @@ private:
HasRepeats = false;
if (Source == ESource::DataSource || Source == ESource::All) {
for (auto& x : Types.DataSources) {
- auto s = HandleProvider(x.Get(), input, output, ctx);
+ auto s = HandleProvider(x.Get(), input, output, ctx);
if (s.Level == TStatus::Error) return s;
HasRepeats = HasRepeats || s == TStatus::Repeat;
@@ -37,7 +37,7 @@ private:
if (Source == ESource::DataSink || Source == ESource::All) {
for (auto& x : Types.DataSinks) {
- auto s = HandleProvider(x.Get(), input, output, ctx);
+ auto s = HandleProvider(x.Get(), input, output, ctx);
if (s.Level == TStatus::Error) return s;
HasRepeats = HasRepeats || s == TStatus::Repeat;
@@ -78,7 +78,7 @@ private:
}
}
- TStatus HandleProvider(IDataProvider* provider, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ TStatus HandleProvider(IDataProvider* provider, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
TExprNode::TPtr newRoot;
TStatus status = GetTransformer(provider).Transform(input, newRoot, ctx);
@@ -89,9 +89,9 @@ private:
}
} else if (status.Level == TStatus::Async) {
PendingProviders.push_back(provider);
- if (newRoot && newRoot != input) {
- output = newRoot;
- }
+ if (newRoot && newRoot != input) {
+ output = newRoot;
+ }
}
return status;
@@ -147,135 +147,135 @@ private:
TExprNode::TListType NewRoots;
};
-template <ESource Source, typename TGetTransformer, typename TFinish>
-class TSpecificDataProposalsInspector : public TGraphTransformerBase {
-public:
- TSpecificDataProposalsInspector(const TTypeAnnotationContext& types, const TString& provider, TGetTransformer getTransformer,
- TFinish finish)
- : Types(types)
- , Provider(provider)
- , GetTransformer(getTransformer)
- , Finish(finish)
- {}
-
-private:
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- bool hasRepeats = false;
- if (Source == ESource::DataSource || Source == ESource::All) {
- if (auto p = Types.DataSourceMap.FindPtr(Provider)) {
- auto s = HandleProvider(p->Get(), input, output, ctx);
- if (s.Level == TStatus::Error) return s;
- hasRepeats = hasRepeats || s == TStatus::Repeat;
- }
- }
-
- if (Source == ESource::DataSink || Source == ESource::All) {
- if (auto p = Types.DataSinkMap.FindPtr(Provider)) {
- auto s = HandleProvider(p->Get(), input, output, ctx);
- if (s.Level == TStatus::Error) return s;
- hasRepeats = hasRepeats || s == TStatus::Repeat;
- }
- }
-
- if (!PendingProviders.empty()) {
- return TStatus::Async;
- }
-
- if (NewRoots.empty()) {
- if (hasRepeats) {
- return TStatus::Repeat;
- }
-
- Finish(ctx);
- return TStatus::Ok;
- }
-
- ChooseRoot(std::move(input), output);
- return TStatus(TStatus::Repeat, true);
- }
-
- void Rewind() final {
- if (Source == ESource::DataSource || Source == ESource::All) {
- if (auto p = Types.DataSourceMap.FindPtr(Provider)) {
- GetTransformer(p->Get()).Rewind();
- }
- }
-
- if (Source == ESource::DataSink || Source == ESource::All) {
- if (auto p = Types.DataSinkMap.FindPtr(Provider)) {
- GetTransformer(p->Get()).Rewind();
- }
- }
- }
-
- TStatus HandleProvider(IDataProvider* provider, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- TExprNode::TPtr newRoot;
- TStatus status = GetTransformer(provider).Transform(input, newRoot, ctx);
-
-
- if (status.Level == TStatus::Ok || status.Level == TStatus::Repeat) {
- if (newRoot && newRoot != input) {
- NewRoots.push_back(newRoot);
- }
- } else if (status.Level == TStatus::Async) {
- PendingProviders.push_back(provider);
- if (newRoot && newRoot != input) {
- output = newRoot;
- }
- }
-
- return status;
- }
-
- NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final {
- TVector<NThreading::TFuture<void>> futures;
- for (auto& x : PendingProviders) {
- futures.push_back(GetTransformer(x).GetAsyncFuture(input));
- }
-
- return WaitExceptionOrAll(futures);
- }
-
- TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- bool hasRepeats = false;
- for (auto& x : PendingProviders) {
- TExprNode::TPtr newRoot;
- TStatus status = GetTransformer(x).ApplyAsyncChanges(input, newRoot, ctx);
- if (status.Level == TStatus::Ok || status.Level == TStatus::Repeat) {
- if (newRoot && newRoot != input) {
- NewRoots.push_back(newRoot);
- }
- hasRepeats = hasRepeats || status == TStatus::Repeat;
- } else {
- return status;
- }
- }
-
- PendingProviders.clear();
- if (NewRoots.empty()) {
- return hasRepeats
- ? TStatus::Repeat
- : TStatus::Ok;
- }
-
- ChooseRoot(std::move(input), output);
- return TStatus(TStatus::Repeat, true);
- }
-
- void ChooseRoot(TExprNode::TPtr&& input, TExprNode::TPtr& output) {
- // just get first
- output = std::move(NewRoots.empty() ? input : NewRoots.front());
- NewRoots.clear();
- }
-
- const TTypeAnnotationContext& Types;
- TString Provider;
- TGetTransformer GetTransformer;
- TFinish Finish;
- TVector<IDataProvider*> PendingProviders;
- TExprNode::TListType NewRoots;
-};
-
+template <ESource Source, typename TGetTransformer, typename TFinish>
+class TSpecificDataProposalsInspector : public TGraphTransformerBase {
+public:
+ TSpecificDataProposalsInspector(const TTypeAnnotationContext& types, const TString& provider, TGetTransformer getTransformer,
+ TFinish finish)
+ : Types(types)
+ , Provider(provider)
+ , GetTransformer(getTransformer)
+ , Finish(finish)
+ {}
+
+private:
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ bool hasRepeats = false;
+ if (Source == ESource::DataSource || Source == ESource::All) {
+ if (auto p = Types.DataSourceMap.FindPtr(Provider)) {
+ auto s = HandleProvider(p->Get(), input, output, ctx);
+ if (s.Level == TStatus::Error) return s;
+ hasRepeats = hasRepeats || s == TStatus::Repeat;
+ }
+ }
+
+ if (Source == ESource::DataSink || Source == ESource::All) {
+ if (auto p = Types.DataSinkMap.FindPtr(Provider)) {
+ auto s = HandleProvider(p->Get(), input, output, ctx);
+ if (s.Level == TStatus::Error) return s;
+ hasRepeats = hasRepeats || s == TStatus::Repeat;
+ }
+ }
+
+ if (!PendingProviders.empty()) {
+ return TStatus::Async;
+ }
+
+ if (NewRoots.empty()) {
+ if (hasRepeats) {
+ return TStatus::Repeat;
+ }
+
+ Finish(ctx);
+ return TStatus::Ok;
+ }
+
+ ChooseRoot(std::move(input), output);
+ return TStatus(TStatus::Repeat, true);
+ }
+
+ void Rewind() final {
+ if (Source == ESource::DataSource || Source == ESource::All) {
+ if (auto p = Types.DataSourceMap.FindPtr(Provider)) {
+ GetTransformer(p->Get()).Rewind();
+ }
+ }
+
+ if (Source == ESource::DataSink || Source == ESource::All) {
+ if (auto p = Types.DataSinkMap.FindPtr(Provider)) {
+ GetTransformer(p->Get()).Rewind();
+ }
+ }
+ }
+
+ TStatus HandleProvider(IDataProvider* provider, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ TExprNode::TPtr newRoot;
+ TStatus status = GetTransformer(provider).Transform(input, newRoot, ctx);
+
+
+ if (status.Level == TStatus::Ok || status.Level == TStatus::Repeat) {
+ if (newRoot && newRoot != input) {
+ NewRoots.push_back(newRoot);
+ }
+ } else if (status.Level == TStatus::Async) {
+ PendingProviders.push_back(provider);
+ if (newRoot && newRoot != input) {
+ output = newRoot;
+ }
+ }
+
+ return status;
+ }
+
+ NThreading::TFuture<void> DoGetAsyncFuture(const TExprNode& input) final {
+ TVector<NThreading::TFuture<void>> futures;
+ for (auto& x : PendingProviders) {
+ futures.push_back(GetTransformer(x).GetAsyncFuture(input));
+ }
+
+ return WaitExceptionOrAll(futures);
+ }
+
+ TStatus DoApplyAsyncChanges(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ bool hasRepeats = false;
+ for (auto& x : PendingProviders) {
+ TExprNode::TPtr newRoot;
+ TStatus status = GetTransformer(x).ApplyAsyncChanges(input, newRoot, ctx);
+ if (status.Level == TStatus::Ok || status.Level == TStatus::Repeat) {
+ if (newRoot && newRoot != input) {
+ NewRoots.push_back(newRoot);
+ }
+ hasRepeats = hasRepeats || status == TStatus::Repeat;
+ } else {
+ return status;
+ }
+ }
+
+ PendingProviders.clear();
+ if (NewRoots.empty()) {
+ return hasRepeats
+ ? TStatus::Repeat
+ : TStatus::Ok;
+ }
+
+ ChooseRoot(std::move(input), output);
+ return TStatus(TStatus::Repeat, true);
+ }
+
+ void ChooseRoot(TExprNode::TPtr&& input, TExprNode::TPtr& output) {
+ // just get first
+ output = std::move(NewRoots.empty() ? input : NewRoots.front());
+ NewRoots.clear();
+ }
+
+ const TTypeAnnotationContext& Types;
+ TString Provider;
+ TGetTransformer GetTransformer;
+ TFinish Finish;
+ TVector<IDataProvider*> PendingProviders;
+ TExprNode::TListType NewRoots;
+};
+
auto DefaultFinish = [](TExprContext&){};
template <ESource Source, typename TGetTransformer, typename TFinish = decltype(DefaultFinish)>
@@ -284,12 +284,12 @@ TAutoPtr<IGraphTransformer> CreateDataProposalsInspector(const TTypeAnnotationCo
return new TDataProposalsInspector<Source, TGetTransformer, TFinish>(types, getTransformer, finish);
}
-template <ESource Source, typename TGetTransformer, typename TFinish = decltype(DefaultFinish)>
-TAutoPtr<IGraphTransformer> CreateSpecificDataProposalsInspector(const TTypeAnnotationContext& types, const TString& provider,
- TGetTransformer getTransformer, TFinish finish = DefaultFinish) {
- return new TSpecificDataProposalsInspector<Source, TGetTransformer, TFinish>(types, provider, getTransformer, finish);
-}
-
+template <ESource Source, typename TGetTransformer, typename TFinish = decltype(DefaultFinish)>
+TAutoPtr<IGraphTransformer> CreateSpecificDataProposalsInspector(const TTypeAnnotationContext& types, const TString& provider,
+ TGetTransformer getTransformer, TFinish finish = DefaultFinish) {
+ return new TSpecificDataProposalsInspector<Source, TGetTransformer, TFinish>(types, provider, getTransformer, finish);
+}
+
} // namespace
TAutoPtr<IGraphTransformer> CreateConfigureTransformer(const TTypeAnnotationContext& types) {
@@ -316,18 +316,18 @@ TAutoPtr<IGraphTransformer> CreateIODiscoveryTransformer(const TTypeAnnotationCo
);
}
-TAutoPtr<IGraphTransformer> CreateEpochsTransformer(const TTypeAnnotationContext& types) {
- return CreateDataProposalsInspector<ESource::DataSource>(
- types,
- [](IDataProvider* provider) -> IGraphTransformer& {
- return provider->GetEpochsTransformer();
- },
- [](TExprContext& ctx) {
- ctx.Step.Done(TExprStep::ELevel::Epochs);
- }
- );
-}
-
+TAutoPtr<IGraphTransformer> CreateEpochsTransformer(const TTypeAnnotationContext& types) {
+ return CreateDataProposalsInspector<ESource::DataSource>(
+ types,
+ [](IDataProvider* provider) -> IGraphTransformer& {
+ return provider->GetEpochsTransformer();
+ },
+ [](TExprContext& ctx) {
+ ctx.Step.Done(TExprStep::ELevel::Epochs);
+ }
+ );
+}
+
TAutoPtr<IGraphTransformer> CreateLogicalDataProposalsInspector(const TTypeAnnotationContext& types) {
return CreateDataProposalsInspector<ESource::DataSink>(
types,
@@ -385,17 +385,17 @@ TAutoPtr<IGraphTransformer> CreatePlanInfoTransformer(const TTypeAnnotationConte
);
}
-TAutoPtr<IGraphTransformer> CreateRecaptureDataProposalsInspector(const TTypeAnnotationContext& types, const TString& provider) {
- return CreateSpecificDataProposalsInspector<ESource::DataSink>(
- types,
- provider,
- [](IDataProvider* provider) -> IGraphTransformer& {
- return provider->GetRecaptureOptProposalTransformer();
- },
- [](TExprContext& ctx) {
- ctx.Step.Done(TExprStep::Recapture);
- }
- );
-}
-
+TAutoPtr<IGraphTransformer> CreateRecaptureDataProposalsInspector(const TTypeAnnotationContext& types, const TString& provider) {
+ return CreateSpecificDataProposalsInspector<ESource::DataSink>(
+ types,
+ provider,
+ [](IDataProvider* provider) -> IGraphTransformer& {
+ return provider->GetRecaptureOptProposalTransformer();
+ },
+ [](TExprContext& ctx) {
+ ctx.Step.Done(TExprStep::Recapture);
+ }
+ );
+}
+
} // namespace NYql
diff --git a/ydb/library/yql/core/yql_opt_proposed_by_data.h b/ydb/library/yql/core/yql_opt_proposed_by_data.h
index 0ec0bcab43..a9a74a06ff 100644
--- a/ydb/library/yql/core/yql_opt_proposed_by_data.h
+++ b/ydb/library/yql/core/yql_opt_proposed_by_data.h
@@ -5,15 +5,15 @@
#include <ydb/library/yql/ast/yql_expr.h>
-#include <util/generic/ptr.h>
-#include <util/generic/string.h>
-
+#include <util/generic/ptr.h>
+#include <util/generic/string.h>
+
namespace NYql {
TAutoPtr<IGraphTransformer> CreateConfigureTransformer(const TTypeAnnotationContext& types);
TAutoPtr<IGraphTransformer> CreateIODiscoveryTransformer(const TTypeAnnotationContext& types);
-TAutoPtr<IGraphTransformer> CreateEpochsTransformer(const TTypeAnnotationContext& types);
-TAutoPtr<IGraphTransformer> CreateRecaptureDataProposalsInspector(const TTypeAnnotationContext& types, const TString& provider);
+TAutoPtr<IGraphTransformer> CreateEpochsTransformer(const TTypeAnnotationContext& types);
+TAutoPtr<IGraphTransformer> CreateRecaptureDataProposalsInspector(const TTypeAnnotationContext& types, const TString& provider);
TAutoPtr<IGraphTransformer> CreateLogicalDataProposalsInspector(const TTypeAnnotationContext& types);
TAutoPtr<IGraphTransformer> CreatePhysicalDataProposalsInspector(const TTypeAnnotationContext& types);
TAutoPtr<IGraphTransformer> CreatePhysicalFinalizers(const TTypeAnnotationContext& types);
diff --git a/ydb/library/yql/core/yql_opt_rewrite_io.cpp b/ydb/library/yql/core/yql_opt_rewrite_io.cpp
index ea96f34b68..b4de4cdb3f 100644
--- a/ydb/library/yql/core/yql_opt_rewrite_io.cpp
+++ b/ydb/library/yql/core/yql_opt_rewrite_io.cpp
@@ -14,15 +14,15 @@ IGraphTransformer::TStatus RewriteIO(const TExprNode::TPtr& input, TExprNode::TP
YQL_ENSURE(node->Type() == TExprNode::Callable);
if (node->Content() == LeftName || node->Content() == RightName) {
auto child = node->Child(0);
- if (child->IsCallable(ReadName)) {
+ if (child->IsCallable(ReadName)) {
auto dataSourceName = child->Child(1)->Child(0)->Content();
- auto datasource = types.DataSourceMap.FindPtr(dataSourceName);
+ auto datasource = types.DataSourceMap.FindPtr(dataSourceName);
YQL_ENSURE(datasource);
return (*datasource)->RewriteIO(node, ctx);
}
} else if (node->IsCallable(WriteName)) {
auto dataSinkName = node->Child(1)->Child(0)->Content();
- auto datasink = types.DataSinkMap.FindPtr(dataSinkName);
+ auto datasink = types.DataSinkMap.FindPtr(dataSinkName);
YQL_ENSURE(datasink);
return (*datasink)->RewriteIO(node, ctx);
}
@@ -39,9 +39,9 @@ IGraphTransformer::TStatus RewriteIO(const TExprNode::TPtr& input, TExprNode::TP
}
for (const auto& ds : types.DataSinks)
- ds->PostRewriteIO();
+ ds->PostRewriteIO();
for (const auto& ds : types.DataSources)
- ds->PostRewriteIO();
+ ds->PostRewriteIO();
ctx.Step.Done(TExprStep::RewriteIO);
return IGraphTransformer::TStatus::Ok;
diff --git a/ydb/library/yql/core/yql_opt_utils.cpp b/ydb/library/yql/core/yql_opt_utils.cpp
index 69aa84a500..f9e59a56bc 100644
--- a/ydb/library/yql/core/yql_opt_utils.cpp
+++ b/ydb/library/yql/core/yql_opt_utils.cpp
@@ -2,13 +2,13 @@
#include "yql_expr_optimize.h"
#include "yql_expr_type_annotation.h"
#include "yql_type_annotation.h"
-#include "yql_type_helpers.h"
+#include "yql_type_helpers.h"
#include <ydb/library/yql/ast/yql_constraint.h>
#include <ydb/library/yql/utils/log/log.h>
#include <util/generic/set.h>
-#include <util/string/type.h>
+#include <util/string/type.h>
namespace NYql {
@@ -73,10 +73,10 @@ bool IsPredicateFlatMap(const TExprNode& node) {
return node.IsCallable({"FlatListIf", "ListIf", "OptionalIf", "FlatOptionalIf"});
}
-bool IsFilterFlatMap(const NNodes::TCoLambda& lambda) {
+bool IsFilterFlatMap(const NNodes::TCoLambda& lambda) {
const auto& arg = lambda.Args().Arg(0);
const auto& body = lambda.Body();
- if (body.Maybe<TCoOptionalIf>() || body.Maybe<TCoListIf>()) {
+ if (body.Maybe<TCoOptionalIf>() || body.Maybe<TCoListIf>()) {
if (body.Ref().Child(1) == arg.Raw()) {
return true;
}
@@ -89,7 +89,7 @@ bool IsListReorder(const TExprNode& node) {
return node.IsCallable({"Sort", "Reverse"});
}
-bool IsRenameFlatMap(const NNodes::TCoFlatMapBase& node, TExprNode::TPtr& structNode) {
+bool IsRenameFlatMap(const NNodes::TCoFlatMapBase& node, TExprNode::TPtr& structNode) {
auto lambda = node.Lambda();
if (!IsJustOrSingleAsList(lambda.Body().Ref())) {
return false;
@@ -123,8 +123,8 @@ bool IsPassthroughLambda(const TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>*
outItem = body.Ref().Child(0);
}
- if (body.Maybe<TCoOptionalIf>() || body.Maybe<TCoListIf>()) {
- outItem = body.Cast<TCoConditionalValueBase>().Value();
+ if (body.Maybe<TCoOptionalIf>() || body.Maybe<TCoListIf>()) {
+ outItem = body.Cast<TCoConditionalValueBase>().Value();
}
if (!outItem) {
@@ -135,13 +135,13 @@ bool IsPassthroughLambda(const TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>*
return true;
}
- if (auto maybeStruct = outItem.Maybe<TCoAsStruct>()) {
+ if (auto maybeStruct = outItem.Maybe<TCoAsStruct>()) {
if (passthroughFields) {
passthroughFields->ConstructInPlace();
}
for (auto child : maybeStruct.Cast()) {
- auto tuple = child.Cast<TCoNameValueTuple>();
+ auto tuple = child.Cast<TCoNameValueTuple>();
auto value = tuple.Value();
if (analyzeJustMember && value.Maybe<TCoJust>()) {
value = value.Cast<TCoJust>().Input();
@@ -176,7 +176,7 @@ bool IsPassthroughLambda(const TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>*
bool IsTablePropsDependent(const TExprNode& node) {
bool found = false;
VisitExpr(node, [&found](const TExprNode& n) {
- found = found || TCoTablePropBase::Match(&n) || (TCoMember::Match(&n) && TCoMember(&n).Name().Value().StartsWith("_yql_sys_"));
+ found = found || TCoTablePropBase::Match(&n) || (TCoMember::Match(&n) && TCoMember(&n).Name().Value().StartsWith("_yql_sys_"));
return !found;
});
return found;
@@ -207,7 +207,7 @@ TExprNode::TPtr KeepColumnOrder(const TExprNode::TPtr& node, const TExprNode& sr
template<class TFieldsSet>
bool HaveFieldsSubset(const TExprNode::TPtr& start, const TExprNode& arg, TFieldsSet& usedFields, const TParentsMap& parentsMap, bool allowDependsOn) {
- if (arg.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
+ if (arg.GetTypeAnn()->GetKind() != ETypeAnnotationKind::Struct) {
return false;
}
@@ -251,11 +251,11 @@ bool HaveFieldsSubset(const TExprNode::TPtr& start, const TExprNode& arg, TField
}
template bool HaveFieldsSubset(const TExprNode::TPtr& start, const TExprNode& arg, TSet<TStringBuf>& usedFields, const TParentsMap& parentsMap, bool allowDependsOn);
-template bool HaveFieldsSubset(const TExprNode::TPtr& start, const TExprNode& arg, TSet<TString>& usedFields, const TParentsMap& parentsMap, bool allowDependsOn);
+template bool HaveFieldsSubset(const TExprNode::TPtr& start, const TExprNode& arg, TSet<TString>& usedFields, const TParentsMap& parentsMap, bool allowDependsOn);
template bool HaveFieldsSubset(const TExprNode::TPtr& start, const TExprNode& arg, std::map<std::string_view, TExprNode::TPtr>& usedFields, const TParentsMap& parentsMap, bool allowDependsOn);
-template<class TFieldsSet>
-TExprNode::TPtr FilterByFields(TPositionHandle position, const TExprNode::TPtr& input, const TFieldsSet& subsetFields,
+template<class TFieldsSet>
+TExprNode::TPtr FilterByFields(TPositionHandle position, const TExprNode::TPtr& input, const TFieldsSet& subsetFields,
TExprContext& ctx, bool singleValue) {
if (singleValue) {
TExprNode::TListType structItems;
@@ -276,10 +276,10 @@ TExprNode::TPtr FilterByFields(TPositionHandle position, const TExprNode::TPtr&
return ctx.NewCallable(position, "ExtractMembers", { input, ctx.NewList(position, std::move(fields)) });
}
-template TExprNode::TPtr FilterByFields(TPositionHandle position, const TExprNode::TPtr& input, const TSet<TStringBuf>& subsetFields, TExprContext& ctx, bool singleValue);
-template TExprNode::TPtr FilterByFields(TPositionHandle position, const TExprNode::TPtr& input, const TSet<TString>& subsetFields, TExprContext& ctx, bool singleValue);
-
-
+template TExprNode::TPtr FilterByFields(TPositionHandle position, const TExprNode::TPtr& input, const TSet<TStringBuf>& subsetFields, TExprContext& ctx, bool singleValue);
+template TExprNode::TPtr FilterByFields(TPositionHandle position, const TExprNode::TPtr& input, const TSet<TString>& subsetFields, TExprContext& ctx, bool singleValue);
+
+
bool IsDependedImpl(const TExprNode* from, const TExprNode* to, TNodeSet& visited) {
if (from == to)
return true;
@@ -301,19 +301,19 @@ bool IsDepended(const TExprNode& from, const TExprNode& to) {
return IsDependedImpl(&from, &to, visited);
}
-bool IsEmpty(const TExprNode& node, const TTypeAnnotationContext& typeCtx) {
- return typeCtx.IsConstraintCheckEnabled<TEmptyConstraintNode>() && node.Type() != TExprNode::Argument && node.GetConstraint<TEmptyConstraintNode>() != nullptr;
+bool IsEmpty(const TExprNode& node, const TTypeAnnotationContext& typeCtx) {
+ return typeCtx.IsConstraintCheckEnabled<TEmptyConstraintNode>() && node.Type() != TExprNode::Argument && node.GetConstraint<TEmptyConstraintNode>() != nullptr;
}
bool IsEmptyOptional(const TExprNode& node) {
return node.IsCallable("Nothing");
}
-bool IsEmptyContainer(const TExprNode& node) {
- return node.IsCallable({"EmptyList", "EmptyDict"})
- || (1U == node.ChildrenSize() && node.IsCallable({"List", "Nothing", "EmptyIterator", "Dict"}));
-}
-
+bool IsEmptyContainer(const TExprNode& node) {
+ return node.IsCallable({"EmptyList", "EmptyDict"})
+ || (1U == node.ChildrenSize() && node.IsCallable({"List", "Nothing", "EmptyIterator", "Dict"}));
+}
+
const TTypeAnnotationNode* RemoveOptionalType(const TTypeAnnotationNode* type) {
if (!type || type->GetKind() != ETypeAnnotationKind::Optional) {
return type;
@@ -514,7 +514,7 @@ TExprNode::TPtr MakeSingleGroupRow(const TExprNode& aggregateNode, TExprNode::TP
bool UpdateStructMembers(TExprContext& ctx, const TExprNode::TPtr& node, const TStringBuf& goal, TExprNode::TListType& members, MemberUpdaterFunc updaterFunc, const TTypeAnnotationNode* nodeType) {
if (!nodeType) {
- nodeType = node->GetTypeAnn();
+ nodeType = node->GetTypeAnn();
Y_VERIFY_DEBUG(nodeType || !"Unset node type for UpdateStructMembers");
}
bool filtered = false;
@@ -524,7 +524,7 @@ bool UpdateStructMembers(TExprContext& ctx, const TExprNode::TPtr& node, const T
const auto memberNode = node->ChildPtr(i);
const auto& memberName = memberNode->Child(0)->Content();
TString useName = TString(memberName);
- if (updaterFunc && !updaterFunc(useName, memberNode->GetTypeAnn())) {
+ if (updaterFunc && !updaterFunc(useName, memberNode->GetTypeAnn())) {
filtered = true;
continue;
}
@@ -578,162 +578,162 @@ TExprNode::TPtr ExpandRemoveMember(const TExprNode::TPtr& node, TExprContext& ct
return ctx.NewCallable(node->Pos(), "AsStruct", std::move(members));
}
-TExprNode::TPtr ExpandRemovePrefixMembers(const TExprNode::TPtr& node, TExprContext& ctx) {
- YQL_CLOG(DEBUG, Core) << "Expand " << node->Content();
-
- const TTypeAnnotationNode* targetItemType = GetItemType(*node->GetTypeAnn());
- bool isSequence = true;
- if (!targetItemType) {
- targetItemType = node->GetTypeAnn();
- isSequence = false;
- }
-
- auto rebuildStruct = [&](const TExprNode::TPtr& srcStruct, const TStructExprType* targetType) -> TExprNode::TPtr {
- TExprNode::TListType nonSystemMembers;
- for (auto item : targetType->GetItems()) {
+TExprNode::TPtr ExpandRemovePrefixMembers(const TExprNode::TPtr& node, TExprContext& ctx) {
+ YQL_CLOG(DEBUG, Core) << "Expand " << node->Content();
+
+ const TTypeAnnotationNode* targetItemType = GetItemType(*node->GetTypeAnn());
+ bool isSequence = true;
+ if (!targetItemType) {
+ targetItemType = node->GetTypeAnn();
+ isSequence = false;
+ }
+
+ auto rebuildStruct = [&](const TExprNode::TPtr& srcStruct, const TStructExprType* targetType) -> TExprNode::TPtr {
+ TExprNode::TListType nonSystemMembers;
+ for (auto item : targetType->GetItems()) {
nonSystemMembers.push_back(
- Build<TCoNameValueTuple>(ctx, srcStruct->Pos())
+ Build<TCoNameValueTuple>(ctx, srcStruct->Pos())
.Name()
.Value(item->GetName())
- .Build()
- .Value<TCoMember>()
- .Struct(srcStruct)
- .Name()
- .Value(item->GetName())
+ .Build()
+ .Value<TCoMember>()
+ .Struct(srcStruct)
+ .Name()
+ .Value(item->GetName())
.Build()
.Build()
- .Done().Ptr()
+ .Done().Ptr()
);
}
- return Build<TCoAsStruct>(ctx, srcStruct->Pos())
+ return Build<TCoAsStruct>(ctx, srcStruct->Pos())
.Add(std::move(nonSystemMembers))
.Done()
.Ptr();
- };
-
- if (targetItemType->GetKind() == ETypeAnnotationKind::Struct) {
- if (isSequence) {
- TExprNode::TListType nonSystemMembers;
- for (auto item : targetItemType->Cast<TStructExprType>()->GetItems()) {
- nonSystemMembers.push_back(ctx.NewAtom(node->Pos(), item->GetName()));
- }
-
- return Build<TCoExtractMembers>(ctx, node->Pos())
- .Input(node->HeadPtr())
- .Members()
- .Add(nonSystemMembers)
- .Build()
- .Done().Ptr();
+ };
+
+ if (targetItemType->GetKind() == ETypeAnnotationKind::Struct) {
+ if (isSequence) {
+ TExprNode::TListType nonSystemMembers;
+ for (auto item : targetItemType->Cast<TStructExprType>()->GetItems()) {
+ nonSystemMembers.push_back(ctx.NewAtom(node->Pos(), item->GetName()));
+ }
+
+ return Build<TCoExtractMembers>(ctx, node->Pos())
+ .Input(node->HeadPtr())
+ .Members()
+ .Add(nonSystemMembers)
+ .Build()
+ .Done().Ptr();
}
- return rebuildStruct(node->HeadPtr(), targetItemType->Cast<TStructExprType>());
- }
-
- YQL_ENSURE(targetItemType->GetKind() == ETypeAnnotationKind::Variant);
-
- const auto targetUnderType = targetItemType->Cast<TVariantExprType>()->GetUnderlyingType();
- TExprNode::TListType variants;
- TTypeAnnotationNode::TListType types;
- switch (targetUnderType->GetKind()) {
- case ETypeAnnotationKind::Tuple: {
- types = targetUnderType->Cast<TTupleExprType>()->GetItems();
- variants.resize(types.size());
- for (ui32 i = 0U; i < variants.size(); ++i) {
- variants[i] = ctx.NewAtom(node->Pos(), ToString(i), TNodeFlags::Default);
- }
- break;
- }
- case ETypeAnnotationKind::Struct: {
- const auto& items = targetUnderType->Cast<TStructExprType>()->GetItems();
- types.resize(items.size());
- variants.resize(items.size());
- for (ui32 i = 0U; i < items.size(); ++i) {
- types[i] = items[i]->GetItemType();
- variants[i] = ctx.NewAtom(node->Pos(), items[i]->GetName());
- }
- break;
- }
- default: break;
- }
- const auto type = ExpandType(node->Pos(), *targetItemType, ctx);
-
- if (isSequence) {
- return ctx.Builder(node->Pos())
- .Callable("Map")
- .Add(0, node->HeadPtr())
- .Lambda(1)
- .Param("varItem")
- .Callable("Visit")
- .Arg(0, "varItem")
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- for (ui32 i = 0U; i < variants.size(); ++i) {
- parent.Add(1 + 2 * i, variants[i]);
- parent.Lambda(2 + 2 * i)
- .Param("item")
- .Callable("Variant")
- .Callable(0, "AsStruct")
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- auto targetStructType = types[i]->Cast<TStructExprType>();
- for (size_t j = 0; j < targetStructType->GetItems().size(); ++j) {
- auto name = targetStructType->GetItems()[j]->GetName();
- parent.List(j)
- .Atom(0, name)
- .Callable(1, "Member")
- .Arg(0, "item")
- .Atom(1, name)
- .Seal()
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Add(1, std::move(variants[i]))
- .Add(2, type)
- .Seal()
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Seal()
- .Seal()
- .Build();
- }
-
- return ctx.Builder(node->Pos())
- .Callable("Visit")
- .Add(0, node->HeadPtr())
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- for (ui32 i = 0U; i < variants.size(); ++i) {
- parent.Add(1 + 2 * i, variants[i]);
- parent.Lambda(2 + 2 * i)
- .Param("item")
- .Callable("Variant")
- .Callable(0, "AsStruct")
- .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
- auto targetStructType = types[i]->Cast<TStructExprType>();
- for (size_t j = 0; j < targetStructType->GetItems().size(); ++j) {
- auto name = targetStructType->GetItems()[j]->GetName();
- parent.List(j)
- .Atom(0, name)
- .Callable(1, "Member")
- .Arg(0, "item")
- .Atom(1, name)
- .Seal()
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Add(1, std::move(variants[i]))
- .Add(2, type)
- .Seal()
- .Seal();
- }
- return parent;
- })
- .Seal()
- .Build();
+ return rebuildStruct(node->HeadPtr(), targetItemType->Cast<TStructExprType>());
+ }
+
+ YQL_ENSURE(targetItemType->GetKind() == ETypeAnnotationKind::Variant);
+
+ const auto targetUnderType = targetItemType->Cast<TVariantExprType>()->GetUnderlyingType();
+ TExprNode::TListType variants;
+ TTypeAnnotationNode::TListType types;
+ switch (targetUnderType->GetKind()) {
+ case ETypeAnnotationKind::Tuple: {
+ types = targetUnderType->Cast<TTupleExprType>()->GetItems();
+ variants.resize(types.size());
+ for (ui32 i = 0U; i < variants.size(); ++i) {
+ variants[i] = ctx.NewAtom(node->Pos(), ToString(i), TNodeFlags::Default);
+ }
+ break;
+ }
+ case ETypeAnnotationKind::Struct: {
+ const auto& items = targetUnderType->Cast<TStructExprType>()->GetItems();
+ types.resize(items.size());
+ variants.resize(items.size());
+ for (ui32 i = 0U; i < items.size(); ++i) {
+ types[i] = items[i]->GetItemType();
+ variants[i] = ctx.NewAtom(node->Pos(), items[i]->GetName());
+ }
+ break;
+ }
+ default: break;
+ }
+ const auto type = ExpandType(node->Pos(), *targetItemType, ctx);
+
+ if (isSequence) {
+ return ctx.Builder(node->Pos())
+ .Callable("Map")
+ .Add(0, node->HeadPtr())
+ .Lambda(1)
+ .Param("varItem")
+ .Callable("Visit")
+ .Arg(0, "varItem")
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ for (ui32 i = 0U; i < variants.size(); ++i) {
+ parent.Add(1 + 2 * i, variants[i]);
+ parent.Lambda(2 + 2 * i)
+ .Param("item")
+ .Callable("Variant")
+ .Callable(0, "AsStruct")
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ auto targetStructType = types[i]->Cast<TStructExprType>();
+ for (size_t j = 0; j < targetStructType->GetItems().size(); ++j) {
+ auto name = targetStructType->GetItems()[j]->GetName();
+ parent.List(j)
+ .Atom(0, name)
+ .Callable(1, "Member")
+ .Arg(0, "item")
+ .Atom(1, name)
+ .Seal()
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Add(1, std::move(variants[i]))
+ .Add(2, type)
+ .Seal()
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Seal()
+ .Seal()
+ .Build();
+ }
+
+ return ctx.Builder(node->Pos())
+ .Callable("Visit")
+ .Add(0, node->HeadPtr())
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ for (ui32 i = 0U; i < variants.size(); ++i) {
+ parent.Add(1 + 2 * i, variants[i]);
+ parent.Lambda(2 + 2 * i)
+ .Param("item")
+ .Callable("Variant")
+ .Callable(0, "AsStruct")
+ .Do([&](TExprNodeBuilder& parent) -> TExprNodeBuilder& {
+ auto targetStructType = types[i]->Cast<TStructExprType>();
+ for (size_t j = 0; j < targetStructType->GetItems().size(); ++j) {
+ auto name = targetStructType->GetItems()[j]->GetName();
+ parent.List(j)
+ .Atom(0, name)
+ .Callable(1, "Member")
+ .Arg(0, "item")
+ .Atom(1, name)
+ .Seal()
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Add(1, std::move(variants[i]))
+ .Add(2, type)
+ .Seal()
+ .Seal();
+ }
+ return parent;
+ })
+ .Seal()
+ .Build();
}
TExprNode::TPtr ExpandFlattenMembers(const TExprNode::TPtr& node, TExprContext& ctx) {
@@ -752,7 +752,7 @@ TExprNode::TPtr ExpandFlattenMembers(const TExprNode::TPtr& node, TExprContext&
TExprNode::TPtr ExpandFlattenStructs(const TExprNode::TPtr& node, TExprContext& ctx) {
TExprNode::TListType members;
auto structObj = node->Child(0);
- for (auto& x : structObj->GetTypeAnn()->Cast<TStructExprType>()->GetItems()) {
+ for (auto& x : structObj->GetTypeAnn()->Cast<TStructExprType>()->GetItems()) {
auto subMember = ctx.Builder(node->Pos())
.Callable("Member")
.Add(0, structObj)
@@ -914,7 +914,7 @@ TExprNode::TPtr ExpandFlattenByColumns(const TExprNode::TPtr& node, TExprContext
}
}
- TString operation = "OrderedMap";
+ TString operation = "OrderedMap";
for (const auto& infoPtr : flattenPriority) {
const TFlattenInfo& flattenInfo = *infoPtr;
auto mapLambda = ctx.NewLambda(node->Pos(), ctx.NewArguments(node->Pos(), { flattenInfo.ArgNode }), std::move(internalResult));
@@ -923,7 +923,7 @@ TExprNode::TPtr ExpandFlattenByColumns(const TExprNode::TPtr& node, TExprContext
.Add(0, flattenInfo.ListMember)
.Add(1, mapLambda)
.Seal().Build();
- operation = "OrderedFlatMap";
+ operation = "OrderedFlatMap";
}
return internalResult;
}
@@ -942,73 +942,73 @@ TExprNode::TPtr ExpandCastStruct(const TExprNode::TPtr& node, TExprContext& ctx)
return ctx.NewCallable(node->Pos(), "AsStruct", std::move(items));
}
-void ExtractSimpleKeys(const TExprNode* keySelectorBody, const TExprNode* keySelectorArg, TVector<TStringBuf>& columns) {
- if (keySelectorBody->IsList()) {
- for (auto& child: keySelectorBody->Children()) {
- if (child->IsCallable("Member") && child->Child(0) == keySelectorArg) {
- columns.push_back(child->Child(1)->Content());
- } else {
- break;
- }
- }
- } else if (keySelectorBody->IsCallable("Member") && keySelectorBody->Child(0) == keySelectorArg) {
- columns.push_back(keySelectorBody->Child(1)->Content());
- }
+void ExtractSimpleKeys(const TExprNode* keySelectorBody, const TExprNode* keySelectorArg, TVector<TStringBuf>& columns) {
+ if (keySelectorBody->IsList()) {
+ for (auto& child: keySelectorBody->Children()) {
+ if (child->IsCallable("Member") && child->Child(0) == keySelectorArg) {
+ columns.push_back(child->Child(1)->Content());
+ } else {
+ break;
+ }
+ }
+ } else if (keySelectorBody->IsCallable("Member") && keySelectorBody->Child(0) == keySelectorArg) {
+ columns.push_back(keySelectorBody->Child(1)->Content());
+ }
}
-
-void ExtractSimpleSortTraits(const TExprNode& sortDirections, const TExprNode& keySelectorLambda, TVector<bool>& dirs, TVector<TStringBuf>& columns) {
- auto keySelectorBody = keySelectorLambda.Child(1);
- auto keySelectorArg = keySelectorLambda.Child(0)->Child(0);
-
- const bool singleKey = sortDirections.IsCallable("Bool") || (sortDirections.IsList() && sortDirections.ChildrenSize() == 1);
- if (singleKey) {
- const auto item = keySelectorBody->IsList() && keySelectorBody->ChildrenSize() == 1
- ? keySelectorBody->Child(0)
- : keySelectorBody;
- if (item->IsCallable("Member") && item->Child(0) == keySelectorArg) {
- columns.push_back(item->Child(1)->Content());
- }
- } else {
- if (keySelectorBody->IsList()) {
- for (size_t i = 0; i < keySelectorBody->ChildrenSize(); ++i) {
- auto& key = keySelectorBody->Children()[i];
- if (key->IsCallable("Member") && key->Child(0) == keySelectorArg) {
- columns.push_back(key->Child(1)->Content());
- } else {
- break;
- }
- }
- }
- }
-
- if (sortDirections.IsList()) {
- for (size_t i = 0; i < sortDirections.ChildrenSize(); ++i) {
- auto& dir = sortDirections.Children()[i];
- if (dir->IsCallable("Bool")) {
- dirs.push_back(IsTrue(dir->Child(0)->Content()));
- } else {
- break;
- }
- }
- } else if (sortDirections.IsCallable("Bool")) {
- dirs.push_back(IsTrue(sortDirections.Child(0)->Content()));
- }
- size_t minSize = Min<size_t>(columns.size(), dirs.size());
- if (columns.size() > minSize) {
- columns.erase(columns.begin() + minSize, columns.end());
- }
- if (dirs.size() > minSize) {
- dirs.erase(dirs.begin() + minSize, dirs.end());
- }
-}
-
+
+void ExtractSimpleSortTraits(const TExprNode& sortDirections, const TExprNode& keySelectorLambda, TVector<bool>& dirs, TVector<TStringBuf>& columns) {
+ auto keySelectorBody = keySelectorLambda.Child(1);
+ auto keySelectorArg = keySelectorLambda.Child(0)->Child(0);
+
+ const bool singleKey = sortDirections.IsCallable("Bool") || (sortDirections.IsList() && sortDirections.ChildrenSize() == 1);
+ if (singleKey) {
+ const auto item = keySelectorBody->IsList() && keySelectorBody->ChildrenSize() == 1
+ ? keySelectorBody->Child(0)
+ : keySelectorBody;
+ if (item->IsCallable("Member") && item->Child(0) == keySelectorArg) {
+ columns.push_back(item->Child(1)->Content());
+ }
+ } else {
+ if (keySelectorBody->IsList()) {
+ for (size_t i = 0; i < keySelectorBody->ChildrenSize(); ++i) {
+ auto& key = keySelectorBody->Children()[i];
+ if (key->IsCallable("Member") && key->Child(0) == keySelectorArg) {
+ columns.push_back(key->Child(1)->Content());
+ } else {
+ break;
+ }
+ }
+ }
+ }
+
+ if (sortDirections.IsList()) {
+ for (size_t i = 0; i < sortDirections.ChildrenSize(); ++i) {
+ auto& dir = sortDirections.Children()[i];
+ if (dir->IsCallable("Bool")) {
+ dirs.push_back(IsTrue(dir->Child(0)->Content()));
+ } else {
+ break;
+ }
+ }
+ } else if (sortDirections.IsCallable("Bool")) {
+ dirs.push_back(IsTrue(sortDirections.Child(0)->Content()));
+ }
+ size_t minSize = Min<size_t>(columns.size(), dirs.size());
+ if (columns.size() > minSize) {
+ columns.erase(columns.begin() + minSize, columns.end());
+ }
+ if (dirs.size() > minSize) {
+ dirs.erase(dirs.begin() + minSize, dirs.end());
+ }
+}
+
const TExprNode& SkipCallables(const TExprNode& node, const std::initializer_list<std::string_view>& skipCallables) {
const TExprNode* p = &node;
while (p->IsCallable(skipCallables)) {
p = &p->Head();
}
return *p;
-}
+}
namespace {
TExprNode::TPtr ApplyWithCastStructForFirstArg(const TExprNode::TPtr& node, const TTypeAnnotationNode& targetType, TExprContext& ctx) {
@@ -1306,94 +1306,94 @@ bool WarnUnroderedSubquery(const TExprNode& unourderedSubquery, TExprContext& ct
return ctx.AddWarning(issue);
}
-TExprNode::TPtr DuplicateIndependentStreams(TExprNode::TPtr lambda, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx) {
- TNodeOnNodeOwnedMap replaces;
- const auto isStream = [] (const TTypeAnnotationNode* type) {
- YQL_ENSURE(type, "Expect type annotated lambda in DuplicateIndependentStreams");
- return type->GetKind() == ETypeAnnotationKind::Stream || type->GetKind() == ETypeAnnotationKind::Flow;
- };
- VisitExpr(lambda->TailPtr(), [&](const TExprNode::TPtr& node) {
- if (stopTraverse(node.Get())) {
- return false;
- }
- if (auto scope = node->GetDependencyScope(); scope.has_value() && scope.value().second == nullptr
- && isStream(node->GetTypeAnn()) && node->IsCallable() && (node->ChildrenSize() == 0 || !isStream(node->Head().GetTypeAnn()))) {
-
- replaces[node.Get()] = ctx.ShallowCopy(*node);
- return false;
- }
- return true;
- });
- return ctx.ReplaceNodes(std::move(lambda), replaces);
-}
-
-IGraphTransformer::TStatus LocalUnorderedOptimize(TExprNode::TPtr input, TExprNode::TPtr& output, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx, TTypeAnnotationContext* typeCtx) {
- output = input;
- TProcessedNodesSet processedNodes;
- bool hasUnordered = false;
- VisitExpr(input, [&stopTraverse, &processedNodes, &hasUnordered](const TExprNode::TPtr& node) {
- if (stopTraverse(node.Get())) {
- processedNodes.insert(node->UniqueId());
- return false;
- }
- else if (TCoUnorderedBase::Match(node.Get())) {
- hasUnordered = true;
- }
- return true;
- });
- if (!hasUnordered) {
- return IGraphTransformer::TStatus::Ok;
- }
-
- TOptimizeExprSettings settings(typeCtx);
- settings.ProcessedNodes = &processedNodes; // Prevent optimizer to go deeper
-
- static THashSet<TStringBuf> CALLABLE = {"AssumeUnique",
- "Map", "OrderedMap",
- "Filter", "OrderedFilter",
- "FlatMap", "OrderedFlatMap",
- "FoldMap", "Fold1Map", "Chain1Map",
- "Take", "Skip",
- "TakeWhile", "SkipWhile",
- "TakeWhileInclusive", "SkipWhileInclusive",
- "SkipNullMembers", "FilterNullMembers",
- "SkipNullElements", "FilterNullElements",
- "Condense", "Condense1",
- "MapJoinCore", "CommonJoinCore",
- "CombineCore", "ExtractMembers",
- "PartitionByKey", "PartitionsByKeys",
- "FromFlow", "ToFlow", "Collect", "Iterator"};
- static THashSet<TStringBuf> SORTED = {"AssumeSorted", "Sort"};
-
- auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
- if (TCoUnorderedBase::Match(node.Get())) {
- if (node->Head().IsCallable(CALLABLE)) {
- auto res = ctx.SwapWithHead(*node);
- auto name = res->Content();
- if (name.SkipPrefix("Ordered")) {
- res = ctx.RenameNode(*res, name);
- }
- return res;
- }
- if (node->Head().IsCallable(SORTED)) {
- return node->Head().HeadPtr();
- }
- }
-
- return node;
- }, ctx, settings);
-
- if (status.Level == IGraphTransformer::TStatus::Error) {
- return status;
- }
-
- if (input != output && input->IsLambda()) {
- output = ctx.DeepCopyLambda(*output);
- status = status.Combine(IGraphTransformer::TStatus::Repeat);
- }
-
- return status;
-
-}
-
+TExprNode::TPtr DuplicateIndependentStreams(TExprNode::TPtr lambda, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx) {
+ TNodeOnNodeOwnedMap replaces;
+ const auto isStream = [] (const TTypeAnnotationNode* type) {
+ YQL_ENSURE(type, "Expect type annotated lambda in DuplicateIndependentStreams");
+ return type->GetKind() == ETypeAnnotationKind::Stream || type->GetKind() == ETypeAnnotationKind::Flow;
+ };
+ VisitExpr(lambda->TailPtr(), [&](const TExprNode::TPtr& node) {
+ if (stopTraverse(node.Get())) {
+ return false;
+ }
+ if (auto scope = node->GetDependencyScope(); scope.has_value() && scope.value().second == nullptr
+ && isStream(node->GetTypeAnn()) && node->IsCallable() && (node->ChildrenSize() == 0 || !isStream(node->Head().GetTypeAnn()))) {
+
+ replaces[node.Get()] = ctx.ShallowCopy(*node);
+ return false;
+ }
+ return true;
+ });
+ return ctx.ReplaceNodes(std::move(lambda), replaces);
}
+
+IGraphTransformer::TStatus LocalUnorderedOptimize(TExprNode::TPtr input, TExprNode::TPtr& output, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx, TTypeAnnotationContext* typeCtx) {
+ output = input;
+ TProcessedNodesSet processedNodes;
+ bool hasUnordered = false;
+ VisitExpr(input, [&stopTraverse, &processedNodes, &hasUnordered](const TExprNode::TPtr& node) {
+ if (stopTraverse(node.Get())) {
+ processedNodes.insert(node->UniqueId());
+ return false;
+ }
+ else if (TCoUnorderedBase::Match(node.Get())) {
+ hasUnordered = true;
+ }
+ return true;
+ });
+ if (!hasUnordered) {
+ return IGraphTransformer::TStatus::Ok;
+ }
+
+ TOptimizeExprSettings settings(typeCtx);
+ settings.ProcessedNodes = &processedNodes; // Prevent optimizer to go deeper
+
+ static THashSet<TStringBuf> CALLABLE = {"AssumeUnique",
+ "Map", "OrderedMap",
+ "Filter", "OrderedFilter",
+ "FlatMap", "OrderedFlatMap",
+ "FoldMap", "Fold1Map", "Chain1Map",
+ "Take", "Skip",
+ "TakeWhile", "SkipWhile",
+ "TakeWhileInclusive", "SkipWhileInclusive",
+ "SkipNullMembers", "FilterNullMembers",
+ "SkipNullElements", "FilterNullElements",
+ "Condense", "Condense1",
+ "MapJoinCore", "CommonJoinCore",
+ "CombineCore", "ExtractMembers",
+ "PartitionByKey", "PartitionsByKeys",
+ "FromFlow", "ToFlow", "Collect", "Iterator"};
+ static THashSet<TStringBuf> SORTED = {"AssumeSorted", "Sort"};
+
+ auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
+ if (TCoUnorderedBase::Match(node.Get())) {
+ if (node->Head().IsCallable(CALLABLE)) {
+ auto res = ctx.SwapWithHead(*node);
+ auto name = res->Content();
+ if (name.SkipPrefix("Ordered")) {
+ res = ctx.RenameNode(*res, name);
+ }
+ return res;
+ }
+ if (node->Head().IsCallable(SORTED)) {
+ return node->Head().HeadPtr();
+ }
+ }
+
+ return node;
+ }, ctx, settings);
+
+ if (status.Level == IGraphTransformer::TStatus::Error) {
+ return status;
+ }
+
+ if (input != output && input->IsLambda()) {
+ output = ctx.DeepCopyLambda(*output);
+ status = status.Combine(IGraphTransformer::TStatus::Repeat);
+ }
+
+ return status;
+
+}
+
+}
diff --git a/ydb/library/yql/core/yql_opt_utils.h b/ydb/library/yql/core/yql_opt_utils.h
index 00635fea81..ca958daf75 100644
--- a/ydb/library/yql/core/yql_opt_utils.h
+++ b/ydb/library/yql/core/yql_opt_utils.h
@@ -1,23 +1,23 @@
#pragma once
-
+
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/core/yql_graph_transformer.h>
-#include <util/generic/hash_set.h>
-#include <util/generic/strbuf.h>
-
-#include <functional>
-
+#include <util/generic/hash_set.h>
+#include <util/generic/strbuf.h>
+
+#include <functional>
+
namespace NYql {
-struct TTypeAnnotationContext;
-
+struct TTypeAnnotationContext;
+
bool IsJustOrSingleAsList(const TExprNode& node);
bool IsTransparentIfPresent(const TExprNode& node);
bool IsPredicateFlatMap(const TExprNode& node);
-bool IsFilterFlatMap(const NNodes::TCoLambda& lambda);
+bool IsFilterFlatMap(const NNodes::TCoLambda& lambda);
bool IsListReorder(const TExprNode& node);
-bool IsRenameFlatMap(const NNodes::TCoFlatMapBase& node, TExprNode::TPtr& structNode);
+bool IsRenameFlatMap(const NNodes::TCoFlatMapBase& node, TExprNode::TPtr& structNode);
bool IsPassthroughFlatMap(const NNodes::TCoFlatMapBase& flatmap, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false);
bool IsPassthroughLambda(const NNodes::TCoLambda& lambda, TMaybe<THashSet<TStringBuf>>* passthroughFields, bool analyzeJustMember = false);
bool IsTablePropsDependent(const TExprNode& node);
@@ -28,14 +28,14 @@ TExprNode::TPtr KeepColumnOrder(const TExprNode::TPtr& node, const TExprNode& sr
template<class TFieldsSet>
bool HaveFieldsSubset(const TExprNode::TPtr& start, const TExprNode& arg, TFieldsSet& usedFields, const TParentsMap& parentsMap, bool allowDependsOn = true);
-template<class TFieldsSet>
-TExprNode::TPtr FilterByFields(TPositionHandle position, const TExprNode::TPtr& input, const TFieldsSet& subsetFields,
+template<class TFieldsSet>
+TExprNode::TPtr FilterByFields(TPositionHandle position, const TExprNode::TPtr& input, const TFieldsSet& subsetFields,
TExprContext& ctx, bool singleValue);
bool IsDepended(const TExprNode& from, const TExprNode& to);
-bool IsEmpty(const TExprNode& node, const TTypeAnnotationContext& typeCtx);
+bool IsEmpty(const TExprNode& node, const TTypeAnnotationContext& typeCtx);
bool IsEmptyOptional(const TExprNode& node);
-bool IsEmptyContainer(const TExprNode& node);
+bool IsEmptyContainer(const TExprNode& node);
const TTypeAnnotationNode* RemoveOptionalType(const TTypeAnnotationNode* type);
const TTypeAnnotationNode* RemoveAllOptionals(const TTypeAnnotationNode* type);
@@ -57,7 +57,7 @@ bool UpdateStructMembers(TExprContext& ctx, const TExprNode::TPtr& node, const T
TExprNode::TPtr MakeSingleGroupRow(const TExprNode& aggregateNode, TExprNode::TPtr reduced, TExprContext& ctx);
TExprNode::TPtr ExpandRemoveMember(const TExprNode::TPtr& node, TExprContext& ctx);
-TExprNode::TPtr ExpandRemovePrefixMembers(const TExprNode::TPtr& node, TExprContext& ctx);
+TExprNode::TPtr ExpandRemovePrefixMembers(const TExprNode::TPtr& node, TExprContext& ctx);
TExprNode::TPtr ExpandFlattenMembers(const TExprNode::TPtr& node, TExprContext& ctx);
TExprNode::TPtr ExpandFlattenStructs(const TExprNode::TPtr& node, TExprContext& ctx);
TExprNode::TPtr ExpandDivePrefixMembers(const TExprNode::TPtr& node, TExprContext& ctx);
@@ -66,12 +66,12 @@ TExprNode::TPtr ExpandReplaceMember(const TExprNode::TPtr& node, TExprContext& c
TExprNode::TPtr ExpandFlattenByColumns(const TExprNode::TPtr& node, TExprContext& ctx);
TExprNode::TPtr ExpandCastStruct(const TExprNode::TPtr& node, TExprContext& ctx);
-void ExtractSimpleKeys(const TExprNode* keySelectorBody, const TExprNode* keySelectorArg, TVector<TStringBuf>& columns);
-inline void ExtractSimpleKeys(const TExprNode& keySelectorLambda, TVector<TStringBuf>& columns) {
- ExtractSimpleKeys(keySelectorLambda.Child(1), keySelectorLambda.Child(0)->Child(0), columns);
-}
-void ExtractSimpleSortTraits(const TExprNode& sortDirections, const TExprNode& keySelectorLambda, TVector<bool>& dirs, TVector<TStringBuf>& columns);
-
+void ExtractSimpleKeys(const TExprNode* keySelectorBody, const TExprNode* keySelectorArg, TVector<TStringBuf>& columns);
+inline void ExtractSimpleKeys(const TExprNode& keySelectorLambda, TVector<TStringBuf>& columns) {
+ ExtractSimpleKeys(keySelectorLambda.Child(1), keySelectorLambda.Child(0)->Child(0), columns);
+}
+void ExtractSimpleSortTraits(const TExprNode& sortDirections, const TExprNode& keySelectorLambda, TVector<bool>& dirs, TVector<TStringBuf>& columns);
+
TExprNode::TPtr MakeNull(TPositionHandle position, TExprContext& ctx);
TExprNode::TPtr MakeConstMap(TPositionHandle position, const TExprNode::TPtr& input, const TExprNode::TPtr& value, TExprContext& ctx);
TExprNode::TPtr MakeBoolNothing(TPositionHandle position, TExprContext& ctx);
@@ -81,7 +81,7 @@ template <bool Bool>
TExprNode::TPtr MakeBool(TPositionHandle position, TExprContext& ctx);
TExprNode::TPtr MakeIdentityLambda(TPositionHandle position, TExprContext& ctx);
-constexpr std::initializer_list<std::string_view> SkippableCallables = {"Unordered", "AssumeSorted", "AssumeUnique", "AssumeColumnOrder", "AssumeAllMembersNullableAtOnce"};
+constexpr std::initializer_list<std::string_view> SkippableCallables = {"Unordered", "AssumeSorted", "AssumeUnique", "AssumeColumnOrder", "AssumeAllMembersNullableAtOnce"};
const TExprNode& SkipCallables(const TExprNode& node, const std::initializer_list<std::string_view>& skipCallables);
@@ -97,8 +97,8 @@ TExprNode::TPtr OptimizeIfPresent(const TExprNode::TPtr& node, TExprContext& ctx
TExprNode::TPtr OptimizeExists(const TExprNode::TPtr& node, TExprContext& ctx);
bool WarnUnroderedSubquery(const TExprNode& unourderedSubquery, TExprContext& ctx);
-TExprNode::TPtr DuplicateIndependentStreams(TExprNode::TPtr lambda, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx);
-IGraphTransformer::TStatus LocalUnorderedOptimize(TExprNode::TPtr input, TExprNode::TPtr& output,
- const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx, TTypeAnnotationContext* typeCtx);
-
+TExprNode::TPtr DuplicateIndependentStreams(TExprNode::TPtr lambda, const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx);
+IGraphTransformer::TStatus LocalUnorderedOptimize(TExprNode::TPtr input, TExprNode::TPtr& output,
+ const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx, TTypeAnnotationContext* typeCtx);
+
}
diff --git a/ydb/library/yql/core/yql_type_annotation.cpp b/ydb/library/yql/core/yql_type_annotation.cpp
index 9f50af3b63..9f0fd0e5f8 100644
--- a/ydb/library/yql/core/yql_type_annotation.cpp
+++ b/ydb/library/yql/core/yql_type_annotation.cpp
@@ -26,13 +26,13 @@ bool TTypeAnnotationContext::Initialize(TExprContext& ctx) {
bool TTypeAnnotationContext::DoInitialize(TExprContext& ctx) {
for (auto& x : DataSources) {
- if (!x->Initialize(ctx)) {
+ if (!x->Initialize(ctx)) {
return false;
}
}
for (auto& x : DataSinks) {
- if (!x->Initialize(ctx)) {
+ if (!x->Initialize(ctx)) {
return false;
}
}
@@ -40,13 +40,13 @@ bool TTypeAnnotationContext::DoInitialize(TExprContext& ctx) {
Y_ENSURE(UserDataStorage);
UserDataStorage->FillUserDataTokens();
- // Disable "in progress" constraints
- //DisableConstraintCheck.emplace(TSortedConstraintNode::Name());
- //DisableConstraintCheck.emplace(TEmptyConstraintNode::Name());
- DisableConstraintCheck.emplace(TUniqueConstraintNode::Name());
- //DisableConstraintCheck.emplace(TMultiConstraintNode::Name());
- //DisableConstraintCheck.emplace(TVarIndexConstraintNode::Name());
-
+ // Disable "in progress" constraints
+ //DisableConstraintCheck.emplace(TSortedConstraintNode::Name());
+ //DisableConstraintCheck.emplace(TEmptyConstraintNode::Name());
+ DisableConstraintCheck.emplace(TUniqueConstraintNode::Name());
+ //DisableConstraintCheck.emplace(TMultiConstraintNode::Name());
+ //DisableConstraintCheck.emplace(TVarIndexConstraintNode::Name());
+
return true;
}
@@ -234,14 +234,14 @@ bool TModuleResolver::AddFromFile(const TStringBuf& file, TExprContext& ctx, ui1
bool isSql = file.EndsWith(".sql");
bool isYql = file.EndsWith(".yql");
if (!isSql && !isYql) {
- ctx.AddError(TIssue(TStringBuilder() << "Unsupported syntax of library file, expected one of (.sql, .yql): " << file));
+ ctx.AddError(TIssue(TStringBuilder() << "Unsupported syntax of library file, expected one of (.sql, .yql): " << file));
return false;
}
const TUserDataBlock* block = UserData->FindUserDataBlock(fullName);
if (!block) {
- ctx.AddError(TIssue(TStringBuilder() << "File not found: " << file));
+ ctx.AddError(TIssue(TStringBuilder() << "File not found: " << file));
return false;
}
@@ -265,8 +265,8 @@ bool TModuleResolver::AddFromFile(const TStringBuf& file, TExprContext& ctx, ui1
break;
case EUserDataType::URL:
if (!UrlLoader) {
- ctx.AddError(TIssue(TStringBuilder() << "Unable to load file \"" << file
- << "\" from url, because url loader is not available"));
+ ctx.AddError(TIssue(TStringBuilder() << "Unable to load file \"" << file
+ << "\" from url, because url loader is not available"));
return false;
}
@@ -289,7 +289,7 @@ bool TModuleResolver::AddFromMemory(const TStringBuf& file, const TString& body,
bool isSql = file.EndsWith(".sql");
bool isYql = file.EndsWith(".yql");
if (!isSql && !isYql) {
- ctx.AddError(TIssue(TStringBuilder() << "Unsupported syntax of library file, expected one of (.sql, .yql): " << file));
+ ctx.AddError(TIssue(TStringBuilder() << "Unsupported syntax of library file, expected one of (.sql, .yql): " << file));
return false;
}
@@ -312,7 +312,7 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu
astRes = ParseAst(body, nullptr, fullName);
if (!astRes.IsOk()) {
ctx.IssueManager.AddIssues(astRes.Issues);
- ctx.AddError(TIssue(TStringBuilder() << "Failed to parse YQL: " << fullName));
+ ctx.AddError(TIssue(TStringBuilder() << "Failed to parse YQL: " << fullName));
return false;
}
} else {
@@ -320,13 +320,13 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu
settings.Mode = NSQLTranslation::ESqlMode::LIBRARY;
settings.File = fullName;
settings.ClusterMapping = ClusterMapping;
- settings.Flags = SqlFlags;
+ settings.Flags = SqlFlags;
settings.SyntaxVersion = syntaxVersion;
settings.V0Behavior = NSQLTranslation::EV0Behavior::Silent;
astRes = SqlToYql(body, settings);
if (!astRes.IsOk()) {
ctx.IssueManager.AddIssues(astRes.Issues);
- ctx.AddError(TIssue(TStringBuilder() << "Failed to parse SQL: " << fullName));
+ ctx.AddError(TIssue(TStringBuilder() << "Failed to parse SQL: " << fullName));
return false;
}
}
@@ -334,14 +334,14 @@ bool TModuleResolver::AddFromMemory(const TString& fullName, const TString& modu
TLibraryCohesion cohesion;
if (!CompileExpr(*astRes.Root, cohesion, LibsContext)) {
ctx.IssueManager.AddIssues(LibsContext.IssueManager.GetIssues());
- ctx.AddError(TIssue(TStringBuilder() << "Failed to compile: " << fullName));
+ ctx.AddError(TIssue(TStringBuilder() << "Failed to compile: " << fullName));
return false;
}
if (OptimizeLibraries) {
if (!OptimizeLibrary(cohesion, LibsContext)) {
ctx.IssueManager.AddIssues(LibsContext.IssueManager.GetIssues());
- ctx.AddError(TIssue(TStringBuilder() << "Failed to optimize: " << fullName));
+ ctx.AddError(TIssue(TStringBuilder() << "Failed to optimize: " << fullName));
return false;
}
}
diff --git a/ydb/library/yql/core/yql_type_annotation.h b/ydb/library/yql/core/yql_type_annotation.h
index 1a1f1fb476..d4914890a1 100644
--- a/ydb/library/yql/core/yql_type_annotation.h
+++ b/ydb/library/yql/core/yql_type_annotation.h
@@ -17,11 +17,11 @@
#include <util/generic/hash.h>
#include <util/generic/hash_set.h>
#include <util/generic/set.h>
-#include <util/generic/vector.h>
+#include <util/generic/vector.h>
#include <util/digest/city.h>
-#include <vector>
-
+#include <vector>
+
namespace NYql {
class IUrlLoader : public TThrRefBase {
@@ -35,13 +35,13 @@ public:
class TModuleResolver : public IModuleResolver {
public:
- TModuleResolver(TModulesTable&& modules, ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
+ TModuleResolver(TModulesTable&& modules, ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
const THashSet<TString>& sqlFlags, bool optimizeLibraries = true, THolder<TExprContext> ownedCtx = {})
: OwnedCtx(std::move(ownedCtx))
, LibsContext(nextUniqueId)
, Modules(std::move(modules))
, ClusterMapping(clusterMapping)
- , SqlFlags(sqlFlags)
+ , SqlFlags(sqlFlags)
, OptimizeLibraries(optimizeLibraries)
{
if (OwnedCtx) {
@@ -49,14 +49,14 @@ public:
}
}
- TModuleResolver(const TModulesTable* parentModules, ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
+ TModuleResolver(const TModulesTable* parentModules, ui64 nextUniqueId, const THashMap<TString, TString>& clusterMapping,
const THashSet<TString>& sqlFlags, bool optimizeLibraries, const TSet<TString>& knownPackages, const THashMap<TString, THashMap<int, TLibraryCohesion>>& libs)
: ParentModules(parentModules)
, LibsContext(nextUniqueId)
, KnownPackages(knownPackages)
, Libs(libs)
, ClusterMapping(clusterMapping)
- , SqlFlags(sqlFlags)
+ , SqlFlags(sqlFlags)
, OptimizeLibraries(optimizeLibraries)
{
}
@@ -105,7 +105,7 @@ private:
THashMap<TString, THashMap<int, TLibraryCohesion>> Libs;
TModulesTable Modules;
const THashMap<TString, TString> ClusterMapping;
- const THashSet<TString> SqlFlags;
+ const THashSet<TString> SqlFlags;
const bool OptimizeLibraries;
THolder<TExprContext::TFreezeGuard> FreezeGuard;
};
@@ -137,13 +137,13 @@ bool SplitUdfName(TStringBuf name, TStringBuf& moduleName, TStringBuf& funcName)
using TUdfModulesTable = THashMap<TString, TString>; // external module name -> alias of file
struct TYqlOperationOptions {
- TString Runner;
+ TString Runner;
TMaybe<TString> AuthenticatedUser;
TMaybe<TString> Id;
TMaybe<TString> SharedId;
- TMaybe<TString> QueryName;
+ TMaybe<TString> QueryName;
TMaybe<TString> Title;
- TMaybe<TString> Url;
+ TMaybe<TString> Url;
TMaybe<TString> AttrsYson;
TMaybe<NYT::TNode> ParametersYson;
};
@@ -175,10 +175,10 @@ private:
struct TTypeAnnotationContext: public TThrRefBase {
TIntrusivePtr<ITimeProvider> TimeProvider;
TIntrusivePtr<IRandomProvider> RandomProvider;
- THashMap<TString, TIntrusivePtr<IDataProvider>> DataSourceMap;
- THashMap<TString, TIntrusivePtr<IDataProvider>> DataSinkMap;
- TVector<TIntrusivePtr<IDataProvider>> DataSources;
- TVector<TIntrusivePtr<IDataProvider>> DataSinks;
+ THashMap<TString, TIntrusivePtr<IDataProvider>> DataSourceMap;
+ THashMap<TString, TIntrusivePtr<IDataProvider>> DataSinkMap;
+ TVector<TIntrusivePtr<IDataProvider>> DataSources;
+ TVector<TIntrusivePtr<IDataProvider>> DataSinks;
TUdfIndex::TPtr UdfIndex;
TUdfIndexPackageSet::TPtr UdfIndexPackageSet;
IUdfResolver::TPtr UdfResolver;
@@ -186,7 +186,7 @@ struct TTypeAnnotationContext: public TThrRefBase {
TUdfModulesTable UdfModules;
TString PureResultDataSource;
TVector<TString> AvailablePureResultDataSources;
- TString FullResultDataSink;
+ TString FullResultDataSink;
TUserDataStorage::TPtr UserDataStorage;
TUserDataTable UserDataStorageCrutches;
TYqlOperationOptions OperationOptions;
@@ -199,13 +199,13 @@ struct TTypeAnnotationContext: public TThrRefBase {
bool IsReadOnly = false;
TAutoPtr<IGraphTransformer> CustomInstantTypeTransformer;
bool Diagnostics = false;
- THashMap<ui64, ui32> NodeToOperationId; // UniqueId->PublicId translation
+ THashMap<ui64, ui32> NodeToOperationId; // UniqueId->PublicId translation
bool EvaluationInProgress = false;
THashMap<ui64, const TTypeAnnotationNode*> ExpectedTypes;
- THashMap<ui64, std::vector<const TConstraintNode*>> ExpectedConstraints;
+ THashMap<ui64, std::vector<const TConstraintNode*>> ExpectedConstraints;
THashMap<ui64, TColumnOrder> ExpectedColumnOrders;
- THashSet<TString> DisableConstraintCheck;
- bool UdfSupportsYield = false;
+ THashSet<TString> DisableConstraintCheck;
+ bool UdfSupportsYield = false;
ui32 EvaluateForLimit = 500;
ui32 EvaluateOrderByColumnLimit = 100;
bool PullUpFlatMapOverJoin = true;
@@ -213,14 +213,14 @@ struct TTypeAnnotationContext: public TThrRefBase {
THashMap<std::tuple<TString, TString, const TTypeAnnotationNode*>,
std::tuple<const TTypeAnnotationNode*, const TTypeAnnotationNode*, const TTypeAnnotationNode*>>
UdfTypeCache; // (name,typecfg,type)->(type,run config type,new user type)
- bool UseTableMetaFromGraph = false;
- bool DiscoveryMode = false;
+ bool UseTableMetaFromGraph = false;
+ bool DiscoveryMode = false;
bool ForceDq = false;
bool DqCaptured = false; // TODO: Add before/after recapture transformers
TString DqFallbackPolicy = "";
- bool StrictTableProps = true;
+ bool StrictTableProps = true;
bool JsonQueryReturnsJsonDocument = false;
- ui32 FolderSubDirsLimit = 1000;
+ ui32 FolderSubDirsLimit = 1000;
// compatibility with v0 or raw s-expression code
bool OrderedColumns = false;
@@ -254,50 +254,50 @@ struct TTypeAnnotationContext: public TThrRefBase {
return *CachedNow;
}
- void AddDataSource(TStringBuf name, TIntrusivePtr<IDataProvider> provider) {
- DataSourceMap[name] = provider;
- DataSources.push_back(std::move(provider));
- }
-
- void AddDataSource(const THashSet<TString>& names, TIntrusivePtr<IDataProvider> provider) {
- for (auto name: names) {
- DataSourceMap[name] = provider;
- }
- DataSources.push_back(std::move(provider));
- }
-
- void AddDataSink(TStringBuf name, TIntrusivePtr<IDataProvider> provider) {
- DataSinkMap[name] = provider;
- DataSinks.push_back(std::move(provider));
- }
-
- void AddDataSink(const THashSet<TString>& names, TIntrusivePtr<IDataProvider> provider) {
- for (auto name: names) {
- DataSinkMap[name] = provider;
- }
- DataSinks.push_back(std::move(provider));
- }
-
+ void AddDataSource(TStringBuf name, TIntrusivePtr<IDataProvider> provider) {
+ DataSourceMap[name] = provider;
+ DataSources.push_back(std::move(provider));
+ }
+
+ void AddDataSource(const THashSet<TString>& names, TIntrusivePtr<IDataProvider> provider) {
+ for (auto name: names) {
+ DataSourceMap[name] = provider;
+ }
+ DataSources.push_back(std::move(provider));
+ }
+
+ void AddDataSink(TStringBuf name, TIntrusivePtr<IDataProvider> provider) {
+ DataSinkMap[name] = provider;
+ DataSinks.push_back(std::move(provider));
+ }
+
+ void AddDataSink(const THashSet<TString>& names, TIntrusivePtr<IDataProvider> provider) {
+ for (auto name: names) {
+ DataSinkMap[name] = provider;
+ }
+ DataSinks.push_back(std::move(provider));
+ }
+
bool Initialize(TExprContext& ctx);
bool DoInitialize(TExprContext& ctx);
const TCredential* FindCredential(const TStringBuf& name) const;
TString FindCredentialContent(const TStringBuf& name1, const TStringBuf& name2, const TString& defaultContent) const;
TString GetDefaultDataSource() const;
-
- TMaybe<ui32> TranslateOperationId(ui64 id) const {
- auto it = NodeToOperationId.find(id);
- return it == NodeToOperationId.end() ? Nothing() : MakeMaybe(it->second);
- }
-
- bool IsConstraintCheckEnabled(TStringBuf name) const {
- return DisableConstraintCheck.find(name) == DisableConstraintCheck.end();
- }
-
- template <class TConstraint>
- bool IsConstraintCheckEnabled() const {
- return DisableConstraintCheck.find(TConstraint::Name()) == DisableConstraintCheck.end();
- }
+
+ TMaybe<ui32> TranslateOperationId(ui64 id) const {
+ auto it = NodeToOperationId.find(id);
+ return it == NodeToOperationId.end() ? Nothing() : MakeMaybe(it->second);
+ }
+
+ bool IsConstraintCheckEnabled(TStringBuf name) const {
+ return DisableConstraintCheck.find(name) == DisableConstraintCheck.end();
+ }
+
+ template <class TConstraint>
+ bool IsConstraintCheckEnabled() const {
+ return DisableConstraintCheck.find(TConstraint::Name()) == DisableConstraintCheck.end();
+ }
};
template <> inline
diff --git a/ydb/library/yql/core/yql_type_helpers.cpp b/ydb/library/yql/core/yql_type_helpers.cpp
index 9ae7cab56f..ace289b0fb 100644
--- a/ydb/library/yql/core/yql_type_helpers.cpp
+++ b/ydb/library/yql/core/yql_type_helpers.cpp
@@ -5,22 +5,22 @@
namespace NYql {
-const TTypeAnnotationNode* GetItemType(const TTypeAnnotationNode& type) {
- switch (type.GetKind()) {
- case ETypeAnnotationKind::List:
- return type.Cast<TListExprType>()->GetItemType();
- case ETypeAnnotationKind::Optional:
- return type.Cast<TOptionalExprType>()->GetItemType();
- case ETypeAnnotationKind::Stream:
- return type.Cast<TStreamExprType>()->GetItemType();
- case ETypeAnnotationKind::Flow:
- return type.Cast<TFlowExprType>()->GetItemType();
- default:
- break;
- }
- return nullptr;
-}
-
+const TTypeAnnotationNode* GetItemType(const TTypeAnnotationNode& type) {
+ switch (type.GetKind()) {
+ case ETypeAnnotationKind::List:
+ return type.Cast<TListExprType>()->GetItemType();
+ case ETypeAnnotationKind::Optional:
+ return type.Cast<TOptionalExprType>()->GetItemType();
+ case ETypeAnnotationKind::Stream:
+ return type.Cast<TStreamExprType>()->GetItemType();
+ case ETypeAnnotationKind::Flow:
+ return type.Cast<TFlowExprType>()->GetItemType();
+ default:
+ break;
+ }
+ return nullptr;
+}
+
TSet<TStringBuf> GetColumnsOfStructOrSequenceOfStruct(const TTypeAnnotationNode& type) {
const TTypeAnnotationNode* itemType = nullptr;
if (type.GetKind() != ETypeAnnotationKind::Struct) {
@@ -44,15 +44,15 @@ bool SilentGetSequenceItemType(TPosition pos, const TTypeAnnotationNode& inputTy
const TTypeAnnotationNode*& result, TIssue& error)
{
result = nullptr;
- const TTypeAnnotationNode* itemType = GetItemType(inputType);
- if (!itemType) {
- error = TIssue(pos, TStringBuilder() << "Expected list, stream, flow or optional, but got: " << inputType);
+ const TTypeAnnotationNode* itemType = GetItemType(inputType);
+ if (!itemType) {
+ error = TIssue(pos, TStringBuilder() << "Expected list, stream, flow or optional, but got: " << inputType);
return false;
}
if (allowMultiIO && itemType->GetKind() != ETypeAnnotationKind::Struct && itemType->GetKind() != ETypeAnnotationKind::Multi) {
if (itemType->GetKind() != ETypeAnnotationKind::Variant) {
- error = TIssue(pos, TStringBuilder() << "Expected Struct or Variant as row type, but got: " << *itemType);
+ error = TIssue(pos, TStringBuilder() << "Expected Struct or Variant as row type, but got: " << *itemType);
return false;
}
auto varType = itemType->Cast<TVariantExprType>()->GetUnderlyingType();
diff --git a/ydb/library/yql/core/yql_type_helpers.h b/ydb/library/yql/core/yql_type_helpers.h
index 593c77592b..701046584b 100644
--- a/ydb/library/yql/core/yql_type_helpers.h
+++ b/ydb/library/yql/core/yql_type_helpers.h
@@ -7,9 +7,9 @@
namespace NYql {
-const TTypeAnnotationNode* GetItemType(const TTypeAnnotationNode& type);
+const TTypeAnnotationNode* GetItemType(const TTypeAnnotationNode& type);
TSet<TStringBuf> GetColumnsOfStructOrSequenceOfStruct(const TTypeAnnotationNode& type);
-
+
const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO);
const TTypeAnnotationNode* GetSequenceItemType(NNodes::TExprBase listNode, bool allowMultiIO, TExprContext& ctx);
const TTypeAnnotationNode* GetSequenceItemType(TPositionHandle pos, const TTypeAnnotationNode* inputType,
diff --git a/ydb/library/yql/dq/opt/dq_opt.cpp b/ydb/library/yql/dq/opt/dq_opt.cpp
index f60f29dd46..704df14138 100644
--- a/ydb/library/yql/dq/opt/dq_opt.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt.cpp
@@ -138,10 +138,10 @@ bool IsSingleConsumer(const TExprBase& node, const TParentsMap& parentsMap) {
return GetConsumersCount(node, parentsMap) == 1;
}
-bool IsSingleConsumerConnection(const TDqConnection& node, const TParentsMap& parentsMap, bool allowStageMultiUsage) {
- return IsSingleConsumer(node, parentsMap)
- && IsSingleConsumer(node.Output(), parentsMap)
- && (allowStageMultiUsage || IsSingleConsumer(node.Output().Stage(), parentsMap));
+bool IsSingleConsumerConnection(const TDqConnection& node, const TParentsMap& parentsMap, bool allowStageMultiUsage) {
+ return IsSingleConsumer(node, parentsMap)
+ && IsSingleConsumer(node.Output(), parentsMap)
+ && (allowStageMultiUsage || IsSingleConsumer(node.Output().Stage(), parentsMap));
}
ui32 GetStageOutputsCount(const TDqStageBase& stage, bool includingSinks) {
diff --git a/ydb/library/yql/dq/opt/dq_opt.h b/ydb/library/yql/dq/opt/dq_opt.h
index 5b3146d0f4..94bd4c5ecc 100644
--- a/ydb/library/yql/dq/opt/dq_opt.h
+++ b/ydb/library/yql/dq/opt/dq_opt.h
@@ -52,7 +52,7 @@ const TNodeMultiSet& GetConsumers(const NNodes::TExprBase& node, const TParentsM
ui32 GetConsumersCount(const NNodes::TExprBase& node, const TParentsMap& parentsMap);
bool IsSingleConsumer(const NNodes::TExprBase& node, const TParentsMap& parentsMap);
-bool IsSingleConsumerConnection(const NNodes::TDqConnection& node, const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
+bool IsSingleConsumerConnection(const NNodes::TDqConnection& node, const TParentsMap& parentsMap, bool allowStageMultiUsage = true);
ui32 GetStageOutputsCount(const NNodes::TDqStageBase& stage, bool includingSinks);
diff --git a/ydb/library/yql/dq/opt/dq_opt_join.cpp b/ydb/library/yql/dq/opt/dq_opt_join.cpp
index 4097b49e24..1e779c5bf5 100644
--- a/ydb/library/yql/dq/opt/dq_opt_join.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_join.cpp
@@ -440,7 +440,7 @@ TExprBase DqPushJoinToStage(const TExprBase node, TExprContext& ctx, IOptimizati
return node;
}
- if (!IsSingleConsumerConnection(connection.Cast(), parentsMap, allowStageMultiUsage)) {
+ if (!IsSingleConsumerConnection(connection.Cast(), parentsMap, allowStageMultiUsage)) {
return node;
}
diff --git a/ydb/library/yql/dq/opt/dq_opt_log.cpp b/ydb/library/yql/dq/opt/dq_opt_log.cpp
index fed0f09f0f..efb890c748 100644
--- a/ydb/library/yql/dq/opt/dq_opt_log.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_log.cpp
@@ -8,7 +8,7 @@
#include <ydb/library/yql/core/yql_opt_utils.h>
#include <ydb/library/yql/core/yql_type_annotation.h>
-
+
using namespace NYql::NNodes;
namespace NYql::NDq {
@@ -189,21 +189,21 @@ NNodes::TExprBase DqMergeQueriesWithSinks(NNodes::TExprBase dqQueryNode, TExprCo
NNodes::TMaybeNode<NNodes::TExprBase> DqUnorderedInStage(NNodes::TExprBase node,
const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx, TTypeAnnotationContext* typeCtx)
{
- auto stage = node.Cast<TDqStageBase>();
-
- TExprNode::TPtr newProgram;
- auto status = LocalUnorderedOptimize(stage.Program().Ptr(), newProgram, stopTraverse, ctx, typeCtx);
- if (status.Level == IGraphTransformer::TStatus::Error) {
- return {};
- }
-
- if (stage.Program().Ptr() != newProgram) {
- return NNodes::TExprBase(ctx.ChangeChild(node.Ref(), TDqStageBase::idx_Program, std::move(newProgram)));
- }
-
- return node;
+ auto stage = node.Cast<TDqStageBase>();
+
+ TExprNode::TPtr newProgram;
+ auto status = LocalUnorderedOptimize(stage.Program().Ptr(), newProgram, stopTraverse, ctx, typeCtx);
+ if (status.Level == IGraphTransformer::TStatus::Error) {
+ return {};
+ }
+
+ if (stage.Program().Ptr() != newProgram) {
+ return NNodes::TExprBase(ctx.ChangeChild(node.Ref(), TDqStageBase::idx_Program, std::move(newProgram)));
+ }
+
+ return node;
}
-
+
NNodes::TExprBase DqFlatMapOverExtend(NNodes::TExprBase node, TExprContext& ctx)
{
auto maybeFlatMap = node.Maybe<TCoFlatMapBase>();
@@ -241,6 +241,6 @@ NNodes::TExprBase DqFlatMapOverExtend(NNodes::TExprBase node, TExprContext& ctx)
auto res = ctx.NewCallable(node.Pos(), extendName, std::move(extendChildren));
return TExprBase(res);
-}
+}
}
diff --git a/ydb/library/yql/dq/opt/dq_opt_log.h b/ydb/library/yql/dq/opt/dq_opt_log.h
index 8e9ed97b06..aed2844d2e 100644
--- a/ydb/library/yql/dq/opt/dq_opt_log.h
+++ b/ydb/library/yql/dq/opt/dq_opt_log.h
@@ -4,12 +4,12 @@
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h>
-#include <functional>
-
-namespace NYql {
- struct TTypeAnnotationContext;
-}
-
+#include <functional>
+
+namespace NYql {
+ struct TTypeAnnotationContext;
+}
+
namespace NYql::NDq {
NNodes::TExprBase DqRewriteAggregate(NNodes::TExprBase node, TExprContext& ctx);
@@ -28,5 +28,5 @@ NNodes::TExprBase DqFlatMapOverExtend(NNodes::TExprBase node, TExprContext& ctx)
NNodes::TMaybeNode<NNodes::TExprBase> DqUnorderedInStage(NNodes::TExprBase node,
const std::function<bool(const TExprNode*)>& stopTraverse, TExprContext& ctx, TTypeAnnotationContext* typeCtx);
-
+
} // namespace NYql::NDq
diff --git a/ydb/library/yql/dq/opt/dq_opt_phy.cpp b/ydb/library/yql/dq/opt/dq_opt_phy.cpp
index cbe871059d..6f46691b32 100644
--- a/ydb/library/yql/dq/opt/dq_opt_phy.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_phy.cpp
@@ -47,8 +47,8 @@ TExprBase DqBuildPartitionsStageStub(TExprBase node, TExprContext& ctx, const TP
!IsDqPureExpr(partition.ListHandlerLambda()) ||
!IsDqPureExpr(partition.SortKeySelectorLambda()))
{
- return node;
- }
+ return node;
+ }
auto dqUnion = partition.Input().template Cast<TDqCnUnionAll>();
if (!IsSingleConsumerConnection(dqUnion, parentsMap)) {
@@ -130,7 +130,7 @@ TExprBase DqBuildPartitionsStageStub(TExprBase node, TExprContext& ctx, const TP
.Args({"rows"})
.Body<TPartition>()
.Input("rows")
- .KeySelectorLambda(ctx.DeepCopyLambda(partition.KeySelectorLambda().Ref()))
+ .KeySelectorLambda(ctx.DeepCopyLambda(partition.KeySelectorLambda().Ref()))
.SortDirections(partition.SortDirections())
.SortKeySelectorLambda(partition.SortKeySelectorLambda().template Maybe<TCoLambda>()
? ctx.DeepCopyLambda(partition.SortKeySelectorLambda().Ref())
@@ -166,7 +166,7 @@ TExprBase DqPushMembersFilterToStage(TExprBase node, TExprContext& ctx, IOptimiz
auto filter = node.Cast<TMembersFilter>();
auto dqUnion = filter.Input().template Cast<TDqCnUnionAll>();
- if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
+ if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
return node;
}
@@ -343,8 +343,8 @@ TExprBase DqBuildFlatmapStage(TExprBase node, TExprContext& ctx, IOptimizationCo
auto flatmap = node.Cast<TCoFlatMapBase>();
auto dqUnion = flatmap.Input().Cast<TDqCnUnionAll>();
if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
- return node;
- }
+ return node;
+ }
if (!IsDqPureExpr(flatmap.Lambda())) {
return node;
@@ -355,7 +355,7 @@ TExprBase DqBuildFlatmapStage(TExprBase node, TExprContext& ctx, IOptimizationCo
.Param("stream")
.Callable(flatmap.Ref().Content())
.Arg(0, "stream")
- .Add(1, ctx.DeepCopyLambda(flatmap.Lambda().Ref()))
+ .Add(1, ctx.DeepCopyLambda(flatmap.Lambda().Ref()))
.Seal()
.Seal().Build());
@@ -393,8 +393,8 @@ TExprBase DqPushBaseLMapToStage(TExprBase node, TExprContext& ctx, IOptimization
auto lmap = node.Cast<BaseLMap>();
auto dqUnion = lmap.Input().template Cast<TDqCnUnionAll>();
if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
- return node;
- }
+ return node;
+ }
if (!CanPushDqExpr(lmap.Lambda(), dqUnion)) {
return node;
@@ -557,8 +557,8 @@ TExprBase DqPushCombineToStage(TExprBase node, TExprContext& ctx, IOptimizationC
auto combine = node.Cast<TCoCombineByKey>();
auto dqUnion = combine.Input().Cast<TDqCnUnionAll>();
if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
- return node;
- }
+ return node;
+ }
if (!CanPushDqExpr(combine.PreMapLambda(), dqUnion) ||
!CanPushDqExpr(combine.KeySelectorLambda(), dqUnion) ||
@@ -573,11 +573,11 @@ TExprBase DqPushCombineToStage(TExprBase node, TExprContext& ctx, IOptimizationC
.Args({"stream"})
.Body<TCoCombineByKey>()
.Input("stream")
- .PreMapLambda(ctx.DeepCopyLambda(combine.PreMapLambda().Ref()))
- .KeySelectorLambda(ctx.DeepCopyLambda(combine.KeySelectorLambda().Ref()))
- .InitHandlerLambda(ctx.DeepCopyLambda(combine.InitHandlerLambda().Ref()))
- .UpdateHandlerLambda(ctx.DeepCopyLambda(combine.UpdateHandlerLambda().Ref()))
- .FinishHandlerLambda(ctx.DeepCopyLambda(combine.FinishHandlerLambda().Ref()))
+ .PreMapLambda(ctx.DeepCopyLambda(combine.PreMapLambda().Ref()))
+ .KeySelectorLambda(ctx.DeepCopyLambda(combine.KeySelectorLambda().Ref()))
+ .InitHandlerLambda(ctx.DeepCopyLambda(combine.InitHandlerLambda().Ref()))
+ .UpdateHandlerLambda(ctx.DeepCopyLambda(combine.UpdateHandlerLambda().Ref()))
+ .FinishHandlerLambda(ctx.DeepCopyLambda(combine.FinishHandlerLambda().Ref()))
.Build()
.Done();
@@ -864,19 +864,19 @@ TExprBase DqBuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationCo
.Args({"stream"})
.Body<TCoTopSort>()
.Input("stream")
- .KeySelectorLambda(ctx.DeepCopyLambda(topSort.KeySelectorLambda().Ref()))
+ .KeySelectorLambda(ctx.DeepCopyLambda(topSort.KeySelectorLambda().Ref()))
.SortDirections(sortDirections)
.Count(topSort.Count())
.Build()
.Done();
- auto stage = dqUnion.Output().Stage().Cast<TDqStage>();
+ auto stage = dqUnion.Output().Stage().Cast<TDqStage>();
auto newStage = DqPushLambdaToStage(stage, dqUnion.Output().Index(), lambda, {}, ctx, optCtx);
- if (!newStage) {
- return node;
- }
-
- bool canMerge = true;
+ if (!newStage) {
+ return node;
+ }
+
+ bool canMerge = true;
auto sortColumnList = Build<TDqSortColumnList>(ctx, node.Pos());
TVector<const TTypeAnnotationNode*> sortKeyTypes;
@@ -890,8 +890,8 @@ TExprBase DqBuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationCo
if (!AddSortColumn(tuple.Item(i), sortDirection, ctx, node, sortKeySelector, sortColumnList,
sortKeyTypes))
{
- canMerge = false;
- break;
+ canMerge = false;
+ break;
}
}
} else {
@@ -899,7 +899,7 @@ TExprBase DqBuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationCo
}
TMaybeNode<TDqStage> outerStage;
- if (canMerge && IsMergeConnectionApplicable(sortKeyTypes)) {
+ if (canMerge && IsMergeConnectionApplicable(sortKeyTypes)) {
auto mergeCn = Build<TDqCnMerge>(ctx, node.Pos())
.Output()
.Stage(newStage.Cast())
@@ -965,7 +965,7 @@ TExprBase DqBuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationConte
auto sort = node.Cast<TCoSortBase>();
auto dqUnion = sort.Input().Cast<TDqCnUnionAll>();
- if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
+ if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
return node;
}
@@ -1007,7 +1007,7 @@ TExprBase DqBuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationConte
.Body<TCoSort>()
.Input("stream")
.SortDirections(sort.SortDirections())
- .KeySelectorLambda(ctx.DeepCopyLambda(sort.KeySelectorLambda().Ref()))
+ .KeySelectorLambda(ctx.DeepCopyLambda(sort.KeySelectorLambda().Ref()))
.Build()
.Done();
@@ -1079,7 +1079,7 @@ TExprBase DqBuildSkipStage(TExprBase node, TExprContext& ctx, IOptimizationConte
}
auto dqUnion = skip.Input().Cast<TDqCnUnionAll>();
- if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
+ if (!IsSingleConsumerConnection(dqUnion, parentsMap, allowStageMultiUsage)) {
return node;
}
diff --git a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp
index a1c8cf10c2..20ee44966c 100644
--- a/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp
+++ b/ydb/library/yql/dq/opt/dq_opt_phy_finalizing.cpp
@@ -4,8 +4,8 @@
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
-#include <algorithm>
-
+#include <algorithm>
+
namespace NYql::NDq {
using namespace NNodes;
@@ -79,11 +79,11 @@ std::pair<TDqStage, TVector<TCoAtom>> ReplicateStageOutput(const TDqStage& stage
}
for (ui32 i = newOutputIndexStart, j = 0; i < newOutputIndexEnd; ++i, ++j) {
- if (outputsCount > 1) {
+ if (outputsCount > 1) {
variants.emplace_back(ctx.FuseLambdas(lambdas[j].Ref(), variants[index].Ref()));
- } else {
+ } else {
variants.emplace_back(ctx.DeepCopyLambda(lambdas[j].Ref()));
- }
+ }
}
newResult = Build<TDqReplicate>(ctx, stage.Pos())
@@ -269,7 +269,7 @@ TExprNode::TPtr ReplicateDqConnection(TExprNode::TPtr&& input, const TMultiUsedC
YQL_ENSURE(consumers.size() > 1);
// NOTE: Only handle one consumer at a time, as there might be dependencies between them.
- // Ensure stable order by processing connection with minimal ID
+ // Ensure stable order by processing connection with minimal ID
auto& consumer = *std::min_element(consumers.begin(), consumers.end(),
[](auto l, auto r) { return l->UniqueId() < r->UniqueId(); });
diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
index 10900352c3..1bcb82a204 100644
--- a/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
+++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.cpp
@@ -264,12 +264,12 @@ public:
auto programRootIdx = programType->FindMemberIndex("Program");
YQL_ENSURE(programRootIdx);
TRuntimeNode programRoot = programStruct.GetValue(*programRootIdx);
- if (Context.FuncProvider) {
- TExploringNodeVisitor explorer;
+ if (Context.FuncProvider) {
+ TExploringNodeVisitor explorer;
explorer.Walk(programRoot.GetNode(), typeEnv);
- bool wereChanges = false;
+ bool wereChanges = false;
programRoot = SinglePassVisitCallables(programRoot, explorer, Context.FuncProvider, typeEnv, true, wereChanges);
- }
+ }
ProgramParsed.OutputItemTypes.resize(task.OutputsSize());
diff --git a/ydb/library/yql/dq/runtime/dq_tasks_runner.h b/ydb/library/yql/dq/runtime/dq_tasks_runner.h
index a0cc816e91..9ae71bc4ff 100644
--- a/ydb/library/yql/dq/runtime/dq_tasks_runner.h
+++ b/ydb/library/yql/dq/runtime/dq_tasks_runner.h
@@ -94,7 +94,7 @@ struct TDqTaskRunnerContext {
TDqComputeContextBase* ComputeCtx = nullptr;
NKikimr::NMiniKQL::TComputationNodeFactory ComputationFactory;
NUdf::IApplyContext* ApplyCtx = nullptr;
- NKikimr::NMiniKQL::TCallableVisitFuncProvider FuncProvider;
+ NKikimr::NMiniKQL::TCallableVisitFuncProvider FuncProvider;
NKikimr::NMiniKQL::TScopedAlloc* Alloc = nullptr;
NKikimr::NMiniKQL::TTypeEnvironment* TypeEnv = nullptr;
};
diff --git a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp
index fcbd3bb9ec..c1c31221ea 100644
--- a/ydb/library/yql/dq/type_ann/dq_type_ann.cpp
+++ b/ydb/library/yql/dq/type_ann/dq_type_ann.cpp
@@ -48,7 +48,7 @@ const TTypeAnnotationNode* GetDqConnectionType(const TDqConnection& node, TExprC
return GetDqOutputType(node.Output(), ctx);
}
-template <typename TStage>
+template <typename TStage>
TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
if (!EnsureMinMaxArgsCount(*stage, 3, 4, ctx)) {
return TStatus::Error;
@@ -62,20 +62,20 @@ TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
return TStatus::Error;
}
- if constexpr (std::is_same_v<TStage, TDqPhyStage>) {
+ if constexpr (std::is_same_v<TStage, TDqPhyStage>) {
if (!EnsureTuple(*settingsTuple, ctx)) {
- return TStatus::Error;
- }
+ return TStatus::Error;
+ }
for (auto& setting: settingsTuple->Children()) {
if (!EnsureTupleMinSize(*setting, 1, ctx)) {
- return TStatus::Error;
- }
+ return TStatus::Error;
+ }
if (!EnsureAtom(*setting->Child(0), ctx)) {
- return TStatus::Error;
- }
- }
- }
-
+ return TStatus::Error;
+ }
+ }
+ }
+
if (!EnsureLambda(*programLambda, ctx)) {
return TStatus::Error;
}
@@ -84,9 +84,9 @@ TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
return TStatus::Error;
}
- TVector<const TTypeAnnotationNode*> argTypes;
+ TVector<const TTypeAnnotationNode*> argTypes;
argTypes.reserve(inputsTuple->ChildrenSize());
-
+
for (const auto& input: inputsTuple->Children()) {
if (!TDqPhyPrecompute::Match(input.Get()) &&
!(TDqConnection::Match(input.Get()) && !TDqCnValue::Match(input.Get())) &&
@@ -106,9 +106,9 @@ TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
} else {
argType = ctx.MakeType<TFlowExprType>(listItemType);
}
- }
+ }
}
- argTypes.emplace_back(argType);
+ argTypes.emplace_back(argType);
}
if (!UpdateLambdaAllArgumentsTypes(programLambda, argTypes, ctx)) {
@@ -117,38 +117,38 @@ TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
auto* resultType = programLambda->GetTypeAnn();
if (!resultType) {
- return TStatus::Repeat;
- }
-
- TVector<const TTypeAnnotationNode*> resultTypesTuple;
-
- if (resultType->GetKind() == ETypeAnnotationKind::Void) {
- // do nothing, return empty tuple as program result
- } else {
- const TTypeAnnotationNode* itemType = nullptr;
- if (resultType->GetKind() == ETypeAnnotationKind::Flow) {
- itemType = resultType->template Cast<TFlowExprType>()->GetItemType();
- } else if (resultType->GetKind() == ETypeAnnotationKind::Stream) {
- itemType = resultType->template Cast<TStreamExprType>()->GetItemType();
- }
- if (itemType) {
- if (itemType->GetKind() == ETypeAnnotationKind::Variant) {
- auto variantType = itemType->Cast<TVariantExprType>()->GetUnderlyingType();
- YQL_ENSURE(variantType->GetKind() == ETypeAnnotationKind::Tuple);
- const auto& items = variantType->Cast<TTupleExprType>()->GetItems();
- resultTypesTuple.reserve(items.size());
- for (const auto* branchType : items) {
- resultTypesTuple.emplace_back(ctx.MakeType<TListExprType>(branchType));
- }
- } else {
- resultTypesTuple.emplace_back(ctx.MakeType<TListExprType>(itemType));
- }
- } else {
+ return TStatus::Repeat;
+ }
+
+ TVector<const TTypeAnnotationNode*> resultTypesTuple;
+
+ if (resultType->GetKind() == ETypeAnnotationKind::Void) {
+ // do nothing, return empty tuple as program result
+ } else {
+ const TTypeAnnotationNode* itemType = nullptr;
+ if (resultType->GetKind() == ETypeAnnotationKind::Flow) {
+ itemType = resultType->template Cast<TFlowExprType>()->GetItemType();
+ } else if (resultType->GetKind() == ETypeAnnotationKind::Stream) {
+ itemType = resultType->template Cast<TStreamExprType>()->GetItemType();
+ }
+ if (itemType) {
+ if (itemType->GetKind() == ETypeAnnotationKind::Variant) {
+ auto variantType = itemType->Cast<TVariantExprType>()->GetUnderlyingType();
+ YQL_ENSURE(variantType->GetKind() == ETypeAnnotationKind::Tuple);
+ const auto& items = variantType->Cast<TTupleExprType>()->GetItems();
+ resultTypesTuple.reserve(items.size());
+ for (const auto* branchType : items) {
+ resultTypesTuple.emplace_back(ctx.MakeType<TListExprType>(branchType));
+ }
+ } else {
+ resultTypesTuple.emplace_back(ctx.MakeType<TListExprType>(itemType));
+ }
+ } else {
YQL_ENSURE(resultType->GetKind() != ETypeAnnotationKind::List, "stage: " << stage->Dump());
- resultTypesTuple.emplace_back(resultType);
+ resultTypesTuple.emplace_back(resultType);
}
}
-
+
if (TDqStageBase::idx_Sinks < stage->ChildrenSize()) {
for (const auto& sink: stage->Child(TDqStageBase::idx_Sinks)->Children()) {
sink->SetTypeAnn(resultType);
@@ -156,7 +156,7 @@ TStatus AnnotateStage(const TExprNode::TPtr& stage, TExprContext& ctx) {
}
stage->SetTypeAnn(ctx.MakeType<TTupleExprType>(resultTypesTuple));
- return TStatus::Ok;
+ return TStatus::Ok;
}
THashMap<TStringBuf, THashMap<TStringBuf, const TTypeAnnotationNode*>>
@@ -293,79 +293,79 @@ const TStructExprType* GetDqJoinResultType(TPositionHandle pos, const TStructExp
}
template <bool IsMapJoin>
-const TStructExprType* GetDqJoinResultType(const TExprNode::TPtr& input, bool stream, TExprContext& ctx) {
- if (!EnsureArgsCount(*input, 6, ctx)) {
- return nullptr;
- }
-
- if (!input->Child(TDqJoin::idx_LeftLabel)->IsCallable("Void")) {
- if (!EnsureAtom(*input->Child(TDqJoin::idx_LeftLabel), ctx)) {
- return nullptr;
- }
- }
-
- if (!input->Child(TDqJoin::idx_RightLabel)->IsCallable("Void")) {
- if (!EnsureAtom(*input->Child(TDqJoin::idx_RightLabel), ctx)) {
- return nullptr;
- }
- }
-
- if (!EnsureAtom(*input->Child(TDqJoin::idx_JoinType), ctx)) {
- return nullptr;
- }
-
- if (!EnsureTuple(*input->Child(TDqJoin::idx_JoinKeys), ctx)) {
- return nullptr;
- }
-
- for (auto& child: input->Child(TDqJoin::idx_JoinKeys)->Children()) {
- if (!EnsureTupleSize(*child, 4, ctx)) {
- return nullptr;
- }
- for (auto& subChild: child->Children()) {
- if (!EnsureAtom(*subChild, ctx)) {
- return nullptr;
- }
- }
- }
-
- auto join = TDqJoinBase(input);
-
- auto leftInputType = join.LeftInput().Ref().GetTypeAnn();
- auto rightInputType = join.RightInput().Ref().GetTypeAnn();
-
+const TStructExprType* GetDqJoinResultType(const TExprNode::TPtr& input, bool stream, TExprContext& ctx) {
+ if (!EnsureArgsCount(*input, 6, ctx)) {
+ return nullptr;
+ }
+
+ if (!input->Child(TDqJoin::idx_LeftLabel)->IsCallable("Void")) {
+ if (!EnsureAtom(*input->Child(TDqJoin::idx_LeftLabel), ctx)) {
+ return nullptr;
+ }
+ }
+
+ if (!input->Child(TDqJoin::idx_RightLabel)->IsCallable("Void")) {
+ if (!EnsureAtom(*input->Child(TDqJoin::idx_RightLabel), ctx)) {
+ return nullptr;
+ }
+ }
+
+ if (!EnsureAtom(*input->Child(TDqJoin::idx_JoinType), ctx)) {
+ return nullptr;
+ }
+
+ if (!EnsureTuple(*input->Child(TDqJoin::idx_JoinKeys), ctx)) {
+ return nullptr;
+ }
+
+ for (auto& child: input->Child(TDqJoin::idx_JoinKeys)->Children()) {
+ if (!EnsureTupleSize(*child, 4, ctx)) {
+ return nullptr;
+ }
+ for (auto& subChild: child->Children()) {
+ if (!EnsureAtom(*subChild, ctx)) {
+ return nullptr;
+ }
+ }
+ }
+
+ auto join = TDqJoinBase(input);
+
+ auto leftInputType = join.LeftInput().Ref().GetTypeAnn();
+ auto rightInputType = join.RightInput().Ref().GetTypeAnn();
+
if (stream) {
- if (!EnsureNewSeqType<false, false, true>(join.Pos(), *leftInputType, ctx)) {
+ if (!EnsureNewSeqType<false, false, true>(join.Pos(), *leftInputType, ctx)) {
return nullptr;
}
- if (!EnsureNewSeqType<false, false, true>(join.Pos(), *rightInputType, ctx)) {
+ if (!EnsureNewSeqType<false, false, true>(join.Pos(), *rightInputType, ctx)) {
return nullptr;
}
} else {
- if (!EnsureNewSeqType<false, true, false>(join.Pos(), *leftInputType, ctx)) {
+ if (!EnsureNewSeqType<false, true, false>(join.Pos(), *leftInputType, ctx)) {
return nullptr;
}
- if (!EnsureNewSeqType<false, true, false>(join.Pos(), *rightInputType, ctx)) {
+ if (!EnsureNewSeqType<false, true, false>(join.Pos(), *rightInputType, ctx)) {
return nullptr;
}
}
auto leftInputItemType = GetSeqItemType(leftInputType);
- if (!EnsureStructType(join.Pos(), *leftInputItemType, ctx)) {
+ if (!EnsureStructType(join.Pos(), *leftInputItemType, ctx)) {
return nullptr;
}
auto leftStructType = leftInputItemType->Cast<TStructExprType>();
- auto leftTableLabel = join.LeftLabel().Maybe<TCoAtom>()
- ? join.LeftLabel().Cast<TCoAtom>().Value()
+ auto leftTableLabel = join.LeftLabel().Maybe<TCoAtom>()
+ ? join.LeftLabel().Cast<TCoAtom>().Value()
: TStringBuf("");
auto rightInputItemType = GetSeqItemType(rightInputType);
- if (!EnsureStructType(join.Pos(), *rightInputItemType, ctx)) {
+ if (!EnsureStructType(join.Pos(), *rightInputItemType, ctx)) {
return nullptr;
}
auto rightStructType = rightInputItemType->Cast<TStructExprType>();
- auto rightTableLabel = join.RightLabel().Maybe<TCoAtom>()
- ? join.RightLabel().Cast<TCoAtom>().Value()
+ auto rightTableLabel = join.RightLabel().Maybe<TCoAtom>()
+ ? join.RightLabel().Cast<TCoAtom>().Value()
: TStringBuf("");
return GetDqJoinResultType<IsMapJoin>(join.Pos(), *leftStructType, leftTableLabel, *rightStructType,
@@ -396,66 +396,66 @@ TStatus AnnotateDqPhyPrecompute(const TExprNode::TPtr& node, TExprContext& ctx)
return TStatus::Ok;
}
-} // unnamed
-
-TStatus AnnotateDqStage(const TExprNode::TPtr& input, TExprContext& ctx) {
- return AnnotateStage<TDqStage>(input, ctx);
-}
-
-TStatus AnnotateDqPhyStage(const TExprNode::TPtr& input, TExprContext& ctx) {
- return AnnotateStage<TDqPhyStage>(input, ctx);
-}
-
-TStatus AnnotateDqOutput(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (!EnsureArgsCount(*input, 2, ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureCallable(*input->Child(TDqOutput::idx_Stage), ctx)) {
- return TStatus::Error;
- }
-
- if (!TDqStageBase::Match(input->Child(TDqOutput::idx_Stage))) {
- ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqOutput::idx_Stage)->Pos()), TStringBuilder() << "Expected " << TDqStage::CallableName() << " or " << TDqPhyStage::CallableName()));
- return TStatus::Error;
- }
-
- if (!EnsureAtom(*input->Child(TDqOutput::idx_Index), ctx)) {
- return TStatus::Error;
- }
-
- auto resultType = GetDqOutputType(TDqOutput(input), ctx);
+} // unnamed
+
+TStatus AnnotateDqStage(const TExprNode::TPtr& input, TExprContext& ctx) {
+ return AnnotateStage<TDqStage>(input, ctx);
+}
+
+TStatus AnnotateDqPhyStage(const TExprNode::TPtr& input, TExprContext& ctx) {
+ return AnnotateStage<TDqPhyStage>(input, ctx);
+}
+
+TStatus AnnotateDqOutput(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (!EnsureArgsCount(*input, 2, ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureCallable(*input->Child(TDqOutput::idx_Stage), ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!TDqStageBase::Match(input->Child(TDqOutput::idx_Stage))) {
+ ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqOutput::idx_Stage)->Pos()), TStringBuilder() << "Expected " << TDqStage::CallableName() << " or " << TDqPhyStage::CallableName()));
+ return TStatus::Error;
+ }
+
+ if (!EnsureAtom(*input->Child(TDqOutput::idx_Index), ctx)) {
+ return TStatus::Error;
+ }
+
+ auto resultType = GetDqOutputType(TDqOutput(input), ctx);
if (!resultType) {
return TStatus::Error;
}
- input->SetTypeAnn(resultType);
- return TStatus::Ok;
-}
-
-TStatus AnnotateDqConnection(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (!EnsureArgsCount(*input, 1, ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureCallable(*input->Child(TDqConnection::idx_Output), ctx)) {
- return TStatus::Error;
- }
-
- if (!TDqOutput::Match(input->Child(TDqConnection::idx_Output))) {
- ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqConnection::idx_Output)->Pos()), TStringBuilder() << "Expected " << TDqOutput::CallableName()));
- return TStatus::Error;
- }
-
- auto resultType = GetDqConnectionType(TDqConnection(input), ctx);
- if (!resultType) {
- return TStatus::Error;
- }
-
- input->SetTypeAnn(resultType);
+ input->SetTypeAnn(resultType);
return TStatus::Ok;
}
+TStatus AnnotateDqConnection(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (!EnsureArgsCount(*input, 1, ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureCallable(*input->Child(TDqConnection::idx_Output), ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!TDqOutput::Match(input->Child(TDqConnection::idx_Output))) {
+ ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqConnection::idx_Output)->Pos()), TStringBuilder() << "Expected " << TDqOutput::CallableName()));
+ return TStatus::Error;
+ }
+
+ auto resultType = GetDqConnectionType(TDqConnection(input), ctx);
+ if (!resultType) {
+ return TStatus::Error;
+ }
+
+ input->SetTypeAnn(resultType);
+ return TStatus::Ok;
+}
+
TStatus AnnotateDqCnMerge(const TExprNode::TPtr& node, TExprContext& ctx) {
if (!EnsureArgsCount(*node, 2, ctx)) {
return TStatus::Error;
@@ -523,199 +523,199 @@ TStatus AnnotateDqCnMerge(const TExprNode::TPtr& node, TExprContext& ctx) {
}
TStatus AnnotateDqCnHashShuffle(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (!EnsureArgsCount(*input, 2, ctx)) {
- return TStatus::Error;
- }
-
+ if (!EnsureArgsCount(*input, 2, ctx)) {
+ return TStatus::Error;
+ }
+
if (!EnsureCallable(*input->Child(TDqCnHashShuffle::idx_Output), ctx)) {
- return TStatus::Error;
- }
-
+ return TStatus::Error;
+ }
+
if (!TDqOutput::Match(input->Child(TDqCnHashShuffle::idx_Output))) {
ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqCnHashShuffle::idx_Output)->Pos()), TStringBuilder() << "Expected " << TDqOutput::CallableName()));
- return TStatus::Error;
- }
-
+ return TStatus::Error;
+ }
+
if (!EnsureTupleMinSize(*input->Child(TDqCnHashShuffle::idx_KeyColumns), 1, ctx)) {
- return TStatus::Error;
- }
-
- auto outputType = GetDqConnectionType(TDqConnection(input), ctx);
- if (!outputType) {
- return TStatus::Error;
- }
-
- auto itemType = outputType->Cast<TListExprType>()->GetItemType();
- if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
- return TStatus::Error;
- }
-
- auto structType = itemType->Cast<TStructExprType>();
+ return TStatus::Error;
+ }
+
+ auto outputType = GetDqConnectionType(TDqConnection(input), ctx);
+ if (!outputType) {
+ return TStatus::Error;
+ }
+
+ auto itemType = outputType->Cast<TListExprType>()->GetItemType();
+ if (!EnsureStructType(input->Pos(), *itemType, ctx)) {
+ return TStatus::Error;
+ }
+
+ auto structType = itemType->Cast<TStructExprType>();
for (const auto& column: input->Child(TDqCnHashShuffle::idx_KeyColumns)->Children()) {
- if (!EnsureAtom(*column, ctx)) {
- return TStatus::Error;
- }
- if (!structType->FindItem(column->Content())) {
- ctx.AddError(TIssue(ctx.GetPosition(column->Pos()),
- TStringBuilder() << "Missing key column: " << column->Content()));
- return TStatus::Error;
- }
- }
-
- input->SetTypeAnn(outputType);
- return TStatus::Ok;
-}
-
+ if (!EnsureAtom(*column, ctx)) {
+ return TStatus::Error;
+ }
+ if (!structType->FindItem(column->Content())) {
+ ctx.AddError(TIssue(ctx.GetPosition(column->Pos()),
+ TStringBuilder() << "Missing key column: " << column->Content()));
+ return TStatus::Error;
+ }
+ }
+
+ input->SetTypeAnn(outputType);
+ return TStatus::Ok;
+}
+
TStatus AnnotateDqCnValue(const TExprNode::TPtr& cnValue, TExprContext& ctx) {
if (!EnsureArgsCount(*cnValue, 1, ctx)) {
- return TStatus::Error;
- }
-
+ return TStatus::Error;
+ }
+
auto& output = cnValue->ChildRef(TDqCnValue::idx_Output);
if (!TDqOutput::Match(output.Get())) {
ctx.AddError(TIssue(ctx.GetPosition(output->Pos()), TStringBuilder() << "Expected " << TDqOutput::CallableName()
<< ", got " << output->Content()));
- return TStatus::Error;
- }
-
+ return TStatus::Error;
+ }
+
auto* resultType = GetDqOutputType(TDqOutput(output), ctx);
- if (!resultType) {
- return TStatus::Error;
- }
-
- const TTypeAnnotationNode* outputType = resultType;
- if (resultType->GetKind() == ETypeAnnotationKind::List) {
- outputType = resultType->Cast<TListExprType>()->GetItemType();
- }
-
+ if (!resultType) {
+ return TStatus::Error;
+ }
+
+ const TTypeAnnotationNode* outputType = resultType;
+ if (resultType->GetKind() == ETypeAnnotationKind::List) {
+ outputType = resultType->Cast<TListExprType>()->GetItemType();
+ }
+
cnValue->SetTypeAnn(outputType);
- return TStatus::Ok;
-}
-
-TStatus AnnotateDqCnResult(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (!EnsureArgsCount(*input, 2, ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureCallable(*input->Child(TDqCnResult::idx_Output), ctx)) {
- return TStatus::Error;
- }
-
- if (!TDqOutput::Match(input->Child(TDqCnResult::idx_Output))) {
- ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqCnResult::idx_Output)->Pos()), TStringBuilder() << "Expected " << TDqOutput::CallableName()));
- return TStatus::Error;
- }
-
- if (!EnsureTuple(*input->Child(TDqCnResult::idx_ColumnHints), ctx)) {
- return TStatus::Error;
- }
-
- for (const auto& column: input->Child(TDqCnResult::idx_ColumnHints)->Children()) {
- if (!EnsureAtom(*column, ctx)) {
- return TStatus::Error;
- }
- }
-
- auto outputType = GetDqConnectionType(TDqConnection(input), ctx);
- if (!outputType) {
- return TStatus::Error;
- }
-
- input->SetTypeAnn(outputType);
- return TStatus::Ok;
-}
-
-TStatus AnnotateDqReplicate(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (!EnsureMinArgsCount(*input, 3, ctx)) {
- return TStatus::Error;
- }
- auto replicateInput = input->Child(TDqReplicate::idx_Input);
- if (!EnsureFlowType(*replicateInput, ctx)) {
- return TStatus::Error;
- }
- auto inputItemType = replicateInput->GetTypeAnn()->Cast<TFlowExprType>()->GetItemType();
- if (!EnsurePersistableType(replicateInput->Pos(), *inputItemType, ctx)) {
- return TStatus::Error;
- }
- if (!EnsureStructType(replicateInput->Pos(), *inputItemType, ctx)) {
- return TStatus::Error;
- }
- const TTypeAnnotationNode* lambdaInputFlowType = ctx.MakeType<TFlowExprType>(inputItemType);
- TVector<const TTypeAnnotationNode*> outputFlowItems;
- for (ui32 i = 1; i < input->ChildrenSize(); ++i) {
- auto& lambda = input->ChildRef(i);
- if (!EnsureLambda(*lambda, ctx)) {
- return TStatus::Error;
- }
- if (!EnsureArgsCount(lambda->Head(), 1, ctx)) {
- return TStatus::Error;
- }
- if (!UpdateLambdaAllArgumentsTypes(lambda, {lambdaInputFlowType}, ctx)) {
- return TStatus::Error;
- }
- if (!lambda->GetTypeAnn()) {
- return TStatus::Repeat;
- }
- if (!EnsureFlowType(lambda->Pos(), *lambda->GetTypeAnn(), ctx)) {
- return TStatus::Error;
- }
- auto lambdaItemType = lambda->GetTypeAnn()->Cast<TFlowExprType>()->GetItemType();
- if (!EnsurePersistableType(lambda->Pos(), *lambdaItemType, ctx)) {
- return TStatus::Error;
- }
- if (!EnsureStructType(lambda->Pos(), *lambdaItemType, ctx)) {
- return TStatus::Error;
- }
- outputFlowItems.push_back(lambdaItemType);
- }
- auto resultItemType = ctx.MakeType<TVariantExprType>(ctx.MakeType<TTupleExprType>(outputFlowItems));
- input->SetTypeAnn(ctx.MakeType<TFlowExprType>(resultItemType));
- return TStatus::Ok;
-}
-
-TStatus AnnotateDqJoin(const TExprNode::TPtr& input, TExprContext& ctx) {
+ return TStatus::Ok;
+}
+
+TStatus AnnotateDqCnResult(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (!EnsureArgsCount(*input, 2, ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureCallable(*input->Child(TDqCnResult::idx_Output), ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!TDqOutput::Match(input->Child(TDqCnResult::idx_Output))) {
+ ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqCnResult::idx_Output)->Pos()), TStringBuilder() << "Expected " << TDqOutput::CallableName()));
+ return TStatus::Error;
+ }
+
+ if (!EnsureTuple(*input->Child(TDqCnResult::idx_ColumnHints), ctx)) {
+ return TStatus::Error;
+ }
+
+ for (const auto& column: input->Child(TDqCnResult::idx_ColumnHints)->Children()) {
+ if (!EnsureAtom(*column, ctx)) {
+ return TStatus::Error;
+ }
+ }
+
+ auto outputType = GetDqConnectionType(TDqConnection(input), ctx);
+ if (!outputType) {
+ return TStatus::Error;
+ }
+
+ input->SetTypeAnn(outputType);
+ return TStatus::Ok;
+}
+
+TStatus AnnotateDqReplicate(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (!EnsureMinArgsCount(*input, 3, ctx)) {
+ return TStatus::Error;
+ }
+ auto replicateInput = input->Child(TDqReplicate::idx_Input);
+ if (!EnsureFlowType(*replicateInput, ctx)) {
+ return TStatus::Error;
+ }
+ auto inputItemType = replicateInput->GetTypeAnn()->Cast<TFlowExprType>()->GetItemType();
+ if (!EnsurePersistableType(replicateInput->Pos(), *inputItemType, ctx)) {
+ return TStatus::Error;
+ }
+ if (!EnsureStructType(replicateInput->Pos(), *inputItemType, ctx)) {
+ return TStatus::Error;
+ }
+ const TTypeAnnotationNode* lambdaInputFlowType = ctx.MakeType<TFlowExprType>(inputItemType);
+ TVector<const TTypeAnnotationNode*> outputFlowItems;
+ for (ui32 i = 1; i < input->ChildrenSize(); ++i) {
+ auto& lambda = input->ChildRef(i);
+ if (!EnsureLambda(*lambda, ctx)) {
+ return TStatus::Error;
+ }
+ if (!EnsureArgsCount(lambda->Head(), 1, ctx)) {
+ return TStatus::Error;
+ }
+ if (!UpdateLambdaAllArgumentsTypes(lambda, {lambdaInputFlowType}, ctx)) {
+ return TStatus::Error;
+ }
+ if (!lambda->GetTypeAnn()) {
+ return TStatus::Repeat;
+ }
+ if (!EnsureFlowType(lambda->Pos(), *lambda->GetTypeAnn(), ctx)) {
+ return TStatus::Error;
+ }
+ auto lambdaItemType = lambda->GetTypeAnn()->Cast<TFlowExprType>()->GetItemType();
+ if (!EnsurePersistableType(lambda->Pos(), *lambdaItemType, ctx)) {
+ return TStatus::Error;
+ }
+ if (!EnsureStructType(lambda->Pos(), *lambdaItemType, ctx)) {
+ return TStatus::Error;
+ }
+ outputFlowItems.push_back(lambdaItemType);
+ }
+ auto resultItemType = ctx.MakeType<TVariantExprType>(ctx.MakeType<TTupleExprType>(outputFlowItems));
+ input->SetTypeAnn(ctx.MakeType<TFlowExprType>(resultItemType));
+ return TStatus::Ok;
+}
+
+TStatus AnnotateDqJoin(const TExprNode::TPtr& input, TExprContext& ctx) {
auto resultRowType = GetDqJoinResultType<false>(input, false, ctx);
- if (!resultRowType) {
- return TStatus::Error;
- }
-
- input->SetTypeAnn(ctx.MakeType<TListExprType>(resultRowType));
- return TStatus::Ok;
-}
-
+ if (!resultRowType) {
+ return TStatus::Error;
+ }
+
+ input->SetTypeAnn(ctx.MakeType<TListExprType>(resultRowType));
+ return TStatus::Ok;
+}
+
TStatus AnnotateDqMapOrDictJoin(const TExprNode::TPtr& input, TExprContext& ctx) {
auto resultRowType = GetDqJoinResultType<true>(input, true, ctx);
- if (!resultRowType) {
- return TStatus::Error;
- }
-
- input->SetTypeAnn(ctx.MakeType<TFlowExprType>(resultRowType));
- return TStatus::Ok;
-}
-
-TStatus AnnotateDqCrossJoin(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (!resultRowType) {
+ return TStatus::Error;
+ }
+
+ input->SetTypeAnn(ctx.MakeType<TFlowExprType>(resultRowType));
+ return TStatus::Ok;
+}
+
+TStatus AnnotateDqCrossJoin(const TExprNode::TPtr& input, TExprContext& ctx) {
auto resultRowType = GetDqJoinResultType<false>(input, true, ctx);
- if (!resultRowType) {
- return TStatus::Error;
- }
-
- auto join = TDqPhyCrossJoin(input);
- if (join.JoinType().Value() != "Cross") {
- ctx.AddError(TIssue(ctx.GetPosition(join.Pos()), TStringBuilder()
- << "Unexpected join type: " << join.JoinType().Value()));
- return TStatus::Error;
- }
-
- if (!join.JoinKeys().Empty()) {
- ctx.AddError(TIssue(ctx.GetPosition(join.Pos()), TStringBuilder()
- << "Expected empty join keys for cross join"));
- return TStatus::Error;
- }
-
- input->SetTypeAnn(ctx.MakeType<TFlowExprType>(resultRowType));
- return TStatus::Ok;
-}
-
+ if (!resultRowType) {
+ return TStatus::Error;
+ }
+
+ auto join = TDqPhyCrossJoin(input);
+ if (join.JoinType().Value() != "Cross") {
+ ctx.AddError(TIssue(ctx.GetPosition(join.Pos()), TStringBuilder()
+ << "Unexpected join type: " << join.JoinType().Value()));
+ return TStatus::Error;
+ }
+
+ if (!join.JoinKeys().Empty()) {
+ ctx.AddError(TIssue(ctx.GetPosition(join.Pos()), TStringBuilder()
+ << "Expected empty join keys for cross join"));
+ return TStatus::Error;
+ }
+
+ input->SetTypeAnn(ctx.MakeType<TFlowExprType>(resultRowType));
+ return TStatus::Ok;
+}
+
TStatus AnnotateDqSource(const TExprNode::TPtr& input, TExprContext& ctx) {
if (!EnsureArgsCount(*input, 2, ctx)) {
return TStatus::Error;
@@ -780,55 +780,55 @@ THolder<IGraphTransformer> CreateDqTypeAnnotationTransformer(TTypeAnnotationCont
TStringBuilder() << "At function: " << input->Content());
});
- if (TDqStage::Match(input.Get())) {
- return AnnotateDqStage(input, ctx);
+ if (TDqStage::Match(input.Get())) {
+ return AnnotateDqStage(input, ctx);
}
- if (TDqPhyStage::Match(input.Get())) {
- return AnnotateDqPhyStage(input, ctx);
+ if (TDqPhyStage::Match(input.Get())) {
+ return AnnotateDqPhyStage(input, ctx);
}
- if (TDqOutput::Match(input.Get())) {
- return AnnotateDqOutput(input, ctx);
+ if (TDqOutput::Match(input.Get())) {
+ return AnnotateDqOutput(input, ctx);
}
- if (TDqCnUnionAll::Match(input.Get())) {
- return AnnotateDqConnection(input, ctx);
+ if (TDqCnUnionAll::Match(input.Get())) {
+ return AnnotateDqConnection(input, ctx);
}
if (TDqCnHashShuffle::Match(input.Get())) {
return AnnotateDqCnHashShuffle(input, ctx);
}
- if (TDqCnMap::Match(input.Get())) {
- return AnnotateDqConnection(input, ctx);
+ if (TDqCnMap::Match(input.Get())) {
+ return AnnotateDqConnection(input, ctx);
}
- if (TDqCnBroadcast::Match(input.Get())) {
- return AnnotateDqConnection(input, ctx);
+ if (TDqCnBroadcast::Match(input.Get())) {
+ return AnnotateDqConnection(input, ctx);
}
- if (TDqCnResult::Match(input.Get())) {
- return AnnotateDqCnResult(input, ctx);
+ if (TDqCnResult::Match(input.Get())) {
+ return AnnotateDqCnResult(input, ctx);
}
- if (TDqCnValue::Match(input.Get())) {
- return AnnotateDqCnValue(input, ctx);
+ if (TDqCnValue::Match(input.Get())) {
+ return AnnotateDqCnValue(input, ctx);
}
if (TDqCnMerge::Match(input.Get())) {
return AnnotateDqCnMerge(input, ctx);
}
- if (TDqReplicate::Match(input.Get())) {
- return AnnotateDqReplicate(input, ctx);
+ if (TDqReplicate::Match(input.Get())) {
+ return AnnotateDqReplicate(input, ctx);
}
- if (TDqJoin::Match(input.Get())) {
- return AnnotateDqJoin(input, ctx);
+ if (TDqJoin::Match(input.Get())) {
+ return AnnotateDqJoin(input, ctx);
}
- if (TDqPhyMapJoin::Match(input.Get())) {
+ if (TDqPhyMapJoin::Match(input.Get())) {
return AnnotateDqMapOrDictJoin(input, ctx);
}
@@ -836,8 +836,8 @@ THolder<IGraphTransformer> CreateDqTypeAnnotationTransformer(TTypeAnnotationCont
return AnnotateDqMapOrDictJoin(input, ctx);
}
- if (TDqPhyCrossJoin::Match(input.Get())) {
- return AnnotateDqCrossJoin(input, ctx);
+ if (TDqPhyCrossJoin::Match(input.Get())) {
+ return AnnotateDqCrossJoin(input, ctx);
}
if (TDqSource::Match(input.Get())) {
diff --git a/ydb/library/yql/dq/type_ann/dq_type_ann.h b/ydb/library/yql/dq/type_ann/dq_type_ann.h
index 03aefe701f..937cc99739 100644
--- a/ydb/library/yql/dq/type_ann/dq_type_ann.h
+++ b/ydb/library/yql/dq/type_ann/dq_type_ann.h
@@ -7,18 +7,18 @@
namespace NYql::NDq {
-IGraphTransformer::TStatus AnnotateDqStage(const TExprNode::TPtr& input, TExprContext& ctx);
-IGraphTransformer::TStatus AnnotateDqPhyStage(const TExprNode::TPtr& input, TExprContext& ctx);
-IGraphTransformer::TStatus AnnotateDqOutput(const TExprNode::TPtr& input, TExprContext& ctx);
+IGraphTransformer::TStatus AnnotateDqStage(const TExprNode::TPtr& input, TExprContext& ctx);
+IGraphTransformer::TStatus AnnotateDqPhyStage(const TExprNode::TPtr& input, TExprContext& ctx);
+IGraphTransformer::TStatus AnnotateDqOutput(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqCnHashShuffle(const TExprNode::TPtr& input, TExprContext& ctx);
-IGraphTransformer::TStatus AnnotateDqCnValue(const TExprNode::TPtr& input, TExprContext& ctx);
-IGraphTransformer::TStatus AnnotateDqCnResult(const TExprNode::TPtr& input, TExprContext& ctx);
-IGraphTransformer::TStatus AnnotateDqReplicate(const TExprNode::TPtr& input, TExprContext& ctx);
-IGraphTransformer::TStatus AnnotateDqConnection(const TExprNode::TPtr& input, TExprContext& ctx);
+IGraphTransformer::TStatus AnnotateDqCnValue(const TExprNode::TPtr& input, TExprContext& ctx);
+IGraphTransformer::TStatus AnnotateDqCnResult(const TExprNode::TPtr& input, TExprContext& ctx);
+IGraphTransformer::TStatus AnnotateDqReplicate(const TExprNode::TPtr& input, TExprContext& ctx);
+IGraphTransformer::TStatus AnnotateDqConnection(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqCnMerge(const TExprNode::TPtr& input, TExprContext& ctx);
-IGraphTransformer::TStatus AnnotateDqJoin(const TExprNode::TPtr& input, TExprContext& ctx);
+IGraphTransformer::TStatus AnnotateDqJoin(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqMapOrDictJoin(const TExprNode::TPtr& input, TExprContext& ctx);
-IGraphTransformer::TStatus AnnotateDqCrossJoin(const TExprNode::TPtr& input, TExprContext& ctx);
+IGraphTransformer::TStatus AnnotateDqCrossJoin(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqSource(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqSink(const TExprNode::TPtr& input, TExprContext& ctx);
IGraphTransformer::TStatus AnnotateDqQuery(const TExprNode::TPtr& input, TExprContext& ctx);
diff --git a/ydb/library/yql/minikql/aligned_page_pool.cpp b/ydb/library/yql/minikql/aligned_page_pool.cpp
index e015c255c8..8269196f12 100644
--- a/ydb/library/yql/minikql/aligned_page_pool.cpp
+++ b/ydb/library/yql/minikql/aligned_page_pool.cpp
@@ -1,48 +1,48 @@
-#include "aligned_page_pool.h"
+#include "aligned_page_pool.h"
#include <library/cpp/actors/util/intrinsics.h>
-
-#include <util/generic/yexception.h>
-#include <util/string/cast.h>
-#include <util/system/align.h>
-#include <util/system/compiler.h>
-#include <util/system/info.h>
-#include <util/system/error.h>
+
+#include <util/generic/yexception.h>
+#include <util/string/cast.h>
+#include <util/system/align.h>
+#include <util/system/compiler.h>
+#include <util/system/info.h>
+#include <util/system/error.h>
#include <util/thread/lfstack.h>
-
-#if defined(_win_)
-#include <util/system/winint.h>
-#elif defined(_unix_)
-#include <sys/types.h>
-#include <sys/mman.h>
-#endif
-
-namespace NKikimr {
-
-static ui64 SYS_PAGE_SIZE = NSystemInfo::GetPageSize();
-
+
+#if defined(_win_)
+#include <util/system/winint.h>
+#elif defined(_unix_)
+#include <sys/types.h>
+#include <sys/mman.h>
+#endif
+
+namespace NKikimr {
+
+static ui64 SYS_PAGE_SIZE = NSystemInfo::GetPageSize();
+
constexpr ui32 MidLevels = 10;
constexpr ui32 MaxMidSize = (1u << MidLevels) * TAlignedPagePool::POOL_PAGE_SIZE;
static_assert(MaxMidSize == 64 * 1024 * 1024, "Upper memory block 64 Mb");
-namespace {
-
-class TGlobalPagePool {
-public:
+namespace {
+
+class TGlobalPagePool {
+public:
TGlobalPagePool(size_t pageSize)
: PageSize(pageSize)
{}
- ~TGlobalPagePool() {
+ ~TGlobalPagePool() {
void* addr = nullptr;
while (Pages.Dequeue(&addr)) {
-#ifdef _win_
- Y_VERIFY_DEBUG(::VirtualFree(addr, 0, MEM_RELEASE), "VirtualFree failed: %s", LastSystemErrorText());
-#else
+#ifdef _win_
+ Y_VERIFY_DEBUG(::VirtualFree(addr, 0, MEM_RELEASE), "VirtualFree failed: %s", LastSystemErrorText());
+#else
Y_VERIFY_DEBUG(0 == ::munmap(addr, PageSize), "munmap failed: %s", LastSystemErrorText());
-#endif
- }
- }
-
+#endif
+ }
+ }
+
void* GetPage() {
void *page = nullptr;
if (Pages.Dequeue(&page)) {
@@ -51,13 +51,13 @@ public:
}
return nullptr;
- }
-
+ }
+
void PushPage(void* addr) {
AtomicIncrement(Count);
Pages.Enqueue(addr);
- }
-
+ }
+
ui64 GetPageCount() const {
return RelaxedLoad(&Count);
}
@@ -70,18 +70,18 @@ public:
return GetPageCount() * GetPageSize();
}
-private:
+private:
const size_t PageSize;
TAtomic Count = 0;
TLockFreeStack<void*> Pages;
-};
-
+};
+
class TGlobalPools {
public:
static TGlobalPools& Instance() {
return *Singleton<TGlobalPools>();
}
-
+
TGlobalPagePool& Get(ui32 index) {
return *Pools[index];
}
@@ -98,8 +98,8 @@ private:
TVector<THolder<TGlobalPagePool>> Pools;
};
-} // unnamed
-
+} // unnamed
+
TAlignedPagePoolCounters::TAlignedPagePoolCounters(NMonitoring::TDynamicCounterPtr countersRoot, const TString& name) {
if (!countersRoot || name.empty())
return;
@@ -110,7 +110,7 @@ TAlignedPagePoolCounters::TAlignedPagePoolCounters(NMonitoring::TDynamicCounterP
LostPagesBytesFreeCntr = subGroup->GetCounter(name + "/LostPagesBytesFreed", true);
}
-TAlignedPagePool::~TAlignedPagePool() {
+TAlignedPagePool::~TAlignedPagePool() {
if (CheckLostMem && !UncaughtException()) {
Y_VERIFY_DEBUG(TotalAllocated == FreePages.size() * POOL_PAGE_SIZE,
"Expected %ld, actual %ld (%ld page(s), %ld offloaded)", TotalAllocated,
@@ -156,9 +156,9 @@ void TAlignedPagePool::ReleaseFreePages() {
for (; !FreePages.empty(); FreePages.pop()) {
AllPages.erase(FreePages.top());
TGlobalPools::Instance().Get(0).PushPage(FreePages.top());
- }
-}
-
+ }
+}
+
void TAlignedPagePool::OffloadAlloc(ui64 size) {
if (Limit && TotalAllocated + size > Limit && !TryIncreaseLimit(TotalAllocated + size)) {
throw TMemoryLimitExceededException();
@@ -197,15 +197,15 @@ void TAlignedPagePool::OffloadFree(ui64 size) noexcept {
}
}
-void* TAlignedPagePool::GetPage() {
- ++PageAllocCount;
- if (!FreePages.empty()) {
- ++PageHitCount;
+void* TAlignedPagePool::GetPage() {
+ ++PageAllocCount;
+ if (!FreePages.empty()) {
+ ++PageHitCount;
const auto res = FreePages.top();
FreePages.pop();
- return res;
- }
-
+ return res;
+ }
+
if (Limit && TotalAllocated + POOL_PAGE_SIZE > Limit && !TryIncreaseLimit(TotalAllocated + POOL_PAGE_SIZE)) {
throw TMemoryLimitExceededException();
}
@@ -218,19 +218,19 @@ void* TAlignedPagePool::GetPage() {
if (Counters.TotalBytesAllocatedCntr) {
(*Counters.TotalBytesAllocatedCntr) += POOL_PAGE_SIZE;
}
- ++PageGlobalHitCount;
+ ++PageGlobalHitCount;
AllPages.emplace(ptr);
UpdatePeaks();
- return ptr;
- }
-
- ++PageMissCount;
+ return ptr;
+ }
+
+ ++PageMissCount;
const auto res = Alloc(POOL_PAGE_SIZE);
AllPages.emplace(res);
return res;
-}
-
+}
+
void TAlignedPagePool::ReturnPage(void* addr) noexcept {
Y_VERIFY_DEBUG(AllPages.find(addr) != AllPages.end());
FreePages.emplace(addr);
@@ -257,9 +257,9 @@ void TAlignedPagePool::ReturnBlock(void* ptr, size_t size) noexcept {
}
}
-void* TAlignedPagePool::Alloc(size_t size) {
- void* res = nullptr;
- size = AlignUp(size, SYS_PAGE_SIZE);
+void* TAlignedPagePool::Alloc(size_t size) {
+ void* res = nullptr;
+ size = AlignUp(size, SYS_PAGE_SIZE);
if (Limit && TotalAllocated + size > Limit && !TryIncreaseLimit(TotalAllocated + size)) {
throw TMemoryLimitExceededException();
@@ -289,9 +289,9 @@ void* TAlignedPagePool::Alloc(size_t size) {
++PageMissCount;
}
}
-
+
if (!res) {
-#ifdef _win_
+#ifdef _win_
res = ::VirtualAlloc(0, size, MEM_RESERVE | MEM_COMMIT, PAGE_READWRITE);
if (Y_UNLIKELY(0 == res)) {
ythrow yexception() << "VirtualAlloc failed to allocate " << size << " bytes: " << LastSystemErrorText();
@@ -304,12 +304,12 @@ void* TAlignedPagePool::Alloc(size_t size) {
if (Counters.TotalBytesAllocatedCntr) {
(*Counters.TotalBytesAllocatedCntr) += size;
}
-#else
+#else
void* mem = ::mmap(nullptr, size + POOL_PAGE_SIZE, PROT_READ | PROT_WRITE, MAP_PRIVATE | MAP_ANON, 0, 0);
if (Y_UNLIKELY(MAP_FAILED == mem)) {
ythrow yexception() << "mmap failed to allocate " << (size + POOL_PAGE_SIZE) << " bytes: " << LastSystemErrorText();
}
-
+
if ((reinterpret_cast<uintptr_t>(mem) & PAGE_ADDR_MASK) == reinterpret_cast<uintptr_t>(mem)) {
// We got already aligned address
res = mem;
@@ -339,39 +339,39 @@ void* TAlignedPagePool::Alloc(size_t size) {
(*Counters.TotalBytesAllocatedCntr) += size;
}
}
- } else {
+ } else {
res = reinterpret_cast<void*>((reinterpret_cast<uintptr_t>(mem) & PAGE_ADDR_MASK) + POOL_PAGE_SIZE);
const size_t off = reinterpret_cast<intptr_t>(res) - reinterpret_cast<intptr_t>(mem);
// Return extra space before/after aligned region to the system
if (Y_UNLIKELY(0 != ::munmap(mem, off))) {
ythrow yexception() << "munmap(0x" << IntToString<16>(reinterpret_cast<uintptr_t>(mem)) << ", " << off
<< ") failed: " << LastSystemErrorText();
- }
+ }
if (Y_UNLIKELY(0 != ::munmap(reinterpret_cast<ui8*>(res) + size, POOL_PAGE_SIZE - off))) {
ythrow yexception() << "munmap(0x" << IntToString<16>(reinterpret_cast<uintptr_t>(res) + size)
<< ", " << (POOL_PAGE_SIZE - off) << ") failed: " << LastSystemErrorText();
}
- TotalAllocated += size;
+ TotalAllocated += size;
if (AllocNotifyCallback) {
AllocNotifyCurrentBytes += size;
}
if (Counters.TotalBytesAllocatedCntr) {
(*Counters.TotalBytesAllocatedCntr) += size;
}
- }
+ }
#endif
- }
+ }
if (Counters.AllocationsCntr) {
++(*Counters.AllocationsCntr);
}
++AllocCount;
UpdatePeaks();
- return res;
-}
-
+ return res;
+}
+
void TAlignedPagePool::Free(void* ptr, size_t size) noexcept {
- size = AlignUp(size, SYS_PAGE_SIZE);
+ size = AlignUp(size, SYS_PAGE_SIZE);
if (size <= MaxMidSize)
size = FastClp2(size);
if (size <= MaxMidSize) {
@@ -379,20 +379,20 @@ void TAlignedPagePool::Free(void* ptr, size_t size) noexcept {
Y_VERIFY_DEBUG(level >= 1 && level <= MidLevels);
TGlobalPools::Instance().Get(level).PushPage(ptr);
} else {
-#ifdef _win_
+#ifdef _win_
Y_VERIFY(::VirtualFree(ptr, 0, MEM_RELEASE));
-#else
+#else
Y_VERIFY(!::munmap(ptr, size));
#endif
- }
+ }
Y_VERIFY_DEBUG(TotalAllocated >= size);
- TotalAllocated -= size;
+ TotalAllocated -= size;
if (Counters.TotalBytesAllocatedCntr) {
(*Counters.TotalBytesAllocatedCntr) -= size;
}
-}
-
+}
+
bool TAlignedPagePool::TryIncreaseLimit(ui64 required) {
if (!IncreaseMemoryLimitCallback) {
return false;
@@ -410,10 +410,10 @@ ui64 TAlignedPagePool::GetGlobalPagePoolSize() {
}
void TAlignedPagePool::PrintStat(size_t usedPages, IOutputStream& out) const {
- usedPages += GetFreePageCount();
- out << "Count of free pages: " << GetFreePageCount() << Endl;
+ usedPages += GetFreePageCount();
+ out << "Count of free pages: " << GetFreePageCount() << Endl;
out << "Allocated for blocks: " << (GetAllocated() - usedPages * TAlignedPagePool::POOL_PAGE_SIZE) << Endl;
- out << "Total allocated by lists: " << GetAllocated() << Endl;
-}
-
-} // NKikimr
+ out << "Total allocated by lists: " << GetAllocated() << Endl;
+}
+
+} // NKikimr
diff --git a/ydb/library/yql/minikql/aligned_page_pool.h b/ydb/library/yql/minikql/aligned_page_pool.h
index 2ef2878655..65f4ec1b90 100644
--- a/ydb/library/yql/minikql/aligned_page_pool.h
+++ b/ydb/library/yql/minikql/aligned_page_pool.h
@@ -1,20 +1,20 @@
-#pragma once
-
+#pragma once
+
#include <util/generic/yexception.h>
-#include <util/stream/output.h>
-#include <util/system/yassert.h>
-#include <util/system/defaults.h>
-
+#include <util/stream/output.h>
+#include <util/system/yassert.h>
+#include <util/system/defaults.h>
+
#include <library/cpp/monlib/dynamic_counters/counters.h>
-#include <type_traits>
+#include <type_traits>
#include <stack>
#include <vector>
#include <unordered_set>
#include <unordered_map>
-
-namespace NKikimr {
-
+
+namespace NKikimr {
+
struct TAlignedPagePoolCounters {
explicit TAlignedPagePoolCounters(NMonitoring::TDynamicCounterPtr countersRoot = nullptr, const TString& name = TString());
@@ -35,11 +35,11 @@ struct TAlignedPagePoolCounters {
// to catch this exception in UDFs code, so we can handle it in the host.
class TMemoryLimitExceededException {};
-class TAlignedPagePool {
-public:
+class TAlignedPagePool {
+public:
static constexpr ui64 POOL_PAGE_SIZE = 1ULL << 16; // 64k
static constexpr ui64 PAGE_ADDR_MASK = ~(POOL_PAGE_SIZE - 1);
-
+
explicit TAlignedPagePool(const TAlignedPagePoolCounters& counters = TAlignedPagePoolCounters())
: Counters(counters)
{
@@ -48,36 +48,36 @@ public:
}
}
- TAlignedPagePool(const TAlignedPagePool&) = delete;
+ TAlignedPagePool(const TAlignedPagePool&) = delete;
TAlignedPagePool(TAlignedPagePool&& other) = delete;
TAlignedPagePool& operator = (const TAlignedPagePool&) = delete;
TAlignedPagePool& operator = (TAlignedPagePool&& other) = delete;
- ~TAlignedPagePool();
-
+ ~TAlignedPagePool();
+
inline size_t GetAllocated() const noexcept {
- return TotalAllocated;
- }
-
+ return TotalAllocated;
+ }
+
inline size_t GetUsed() const noexcept {
return TotalAllocated - GetFreePageCount() * POOL_PAGE_SIZE;
}
inline size_t GetFreePageCount() const noexcept {
- return FreePages.size();
- }
-
+ return FreePages.size();
+ }
+
static inline const void* GetPageStart(const void* addr) noexcept {
- return reinterpret_cast<const void*>(reinterpret_cast<uintptr_t>(addr) & PAGE_ADDR_MASK);
- }
-
+ return reinterpret_cast<const void*>(reinterpret_cast<uintptr_t>(addr) & PAGE_ADDR_MASK);
+ }
+
static inline void* GetPageStart(void* addr) noexcept {
- return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(addr) & PAGE_ADDR_MASK);
- }
-
- void* GetPage();
-
+ return reinterpret_cast<void*>(reinterpret_cast<uintptr_t>(addr) & PAGE_ADDR_MASK);
+ }
+
+ void* GetPage();
+
void ReturnPage(void* addr) noexcept;
void Swap(TAlignedPagePool& other) {
@@ -85,14 +85,14 @@ public:
DoSwap(AllPages, other.AllPages);
DoSwap(ActiveBlocks, other.ActiveBlocks);
DoSwap(TotalAllocated, other.TotalAllocated);
- DoSwap(PeakAllocated, other.PeakAllocated);
- DoSwap(PeakUsed, other.PeakUsed);
+ DoSwap(PeakAllocated, other.PeakAllocated);
+ DoSwap(PeakUsed, other.PeakUsed);
DoSwap(Limit, other.Limit);
- DoSwap(AllocCount, other.AllocCount);
- DoSwap(PageAllocCount, other.PageAllocCount);
- DoSwap(PageHitCount, other.PageHitCount);
- DoSwap(PageGlobalHitCount, other.PageGlobalHitCount);
- DoSwap(PageMissCount, other.PageMissCount);
+ DoSwap(AllocCount, other.AllocCount);
+ DoSwap(PageAllocCount, other.PageAllocCount);
+ DoSwap(PageHitCount, other.PageHitCount);
+ DoSwap(PageGlobalHitCount, other.PageGlobalHitCount);
+ DoSwap(PageMissCount, other.PageMissCount);
DoSwap(OffloadedAllocCount, other.OffloadedAllocCount);
DoSwap(OffloadedBytes, other.OffloadedBytes);
DoSwap(OffloadedActiveBytes, other.OffloadedActiveBytes);
@@ -103,7 +103,7 @@ public:
}
void PrintStat(size_t usedPages, IOutputStream& out) const;
-
+
void* GetBlock(size_t size);
void ReturnBlock(void* ptr, size_t size) noexcept;
@@ -129,9 +129,9 @@ public:
}
ui64 GetPageGlobalHitCount() const noexcept {
- return PageGlobalHitCount;
- }
-
+ return PageGlobalHitCount;
+ }
+
ui64 GetPageMissCount() const noexcept {
return PageMissCount;
}
@@ -176,10 +176,10 @@ public:
IncreaseMemoryLimitCallback = std::move(callback);
}
-protected:
- void* Alloc(size_t size);
+protected:
+ void* Alloc(size_t size);
void Free(void* ptr, size_t size) noexcept;
-
+
void UpdatePeaks() {
PeakAllocated = Max(PeakAllocated, GetAllocated());
PeakUsed = Max(PeakUsed, GetUsed());
@@ -187,11 +187,11 @@ protected:
bool TryIncreaseLimit(ui64 required);
-protected:
+protected:
std::stack<void*, std::vector<void*>> FreePages;
std::unordered_set<void*> AllPages;
std::unordered_map<void*, size_t> ActiveBlocks;
- size_t TotalAllocated = 0;
+ size_t TotalAllocated = 0;
size_t PeakAllocated = 0;
size_t PeakUsed = 0;
size_t Limit = 0;
@@ -199,9 +199,9 @@ protected:
ui64 AllocCount = 0;
ui64 PageAllocCount = 0;
ui64 PageHitCount = 0;
- ui64 PageGlobalHitCount = 0;
+ ui64 PageGlobalHitCount = 0;
ui64 PageMissCount = 0;
-
+
ui64 OffloadedAllocCount = 0;
ui64 OffloadedBytes = 0;
ui64 OffloadedActiveBytes = 0;
@@ -214,6 +214,6 @@ protected:
ui64 AllocNotifyCurrentBytes = 0;
TIncreaseMemoryLimitCallback IncreaseMemoryLimitCallback;
-};
-
-} // NKikimr
+};
+
+} // NKikimr
diff --git a/ydb/library/yql/minikql/benchmark/pack_num/metrics/main.py b/ydb/library/yql/minikql/benchmark/pack_num/metrics/main.py
index 0cead0dad3..b029ea8957 100644
--- a/ydb/library/yql/minikql/benchmark/pack_num/metrics/main.py
+++ b/ydb/library/yql/minikql/benchmark/pack_num/metrics/main.py
@@ -1,7 +1,7 @@
-import yatest.common as yc
-
-
-def test_export_metrics(metrics):
- metrics.set_benchmark(yc.execute_benchmark(
+import yatest.common as yc
+
+
+def test_export_metrics(metrics):
+ metrics.set_benchmark(yc.execute_benchmark(
'yql/minikql/benchmark/pack_num/pack_num',
- threads=8))
+ threads=8))
diff --git a/ydb/library/yql/minikql/benchmark/pack_num/metrics/ya.make b/ydb/library/yql/minikql/benchmark/pack_num/metrics/ya.make
index 203af2de49..c24038588d 100644
--- a/ydb/library/yql/minikql/benchmark/pack_num/metrics/ya.make
+++ b/ydb/library/yql/minikql/benchmark/pack_num/metrics/ya.make
@@ -1,25 +1,25 @@
-OWNER(
+OWNER(
g:kikimr
g:yql
- udovichenko-r
-)
-
+ udovichenko-r
+)
+
PY2TEST()
-
+
SIZE(LARGE)
-
-TAG(
+
+TAG(
ya:force_sandbox
- sb:intel_e5_2660v1
+ sb:intel_e5_2660v1
ya:fat
-)
-
-TEST_SRCS(
- main.py
-)
-
-DEPENDS(
+)
+
+TEST_SRCS(
+ main.py
+)
+
+DEPENDS(
ydb/library/yql/minikql/benchmark/pack_num
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/minikql/benchmark/pack_num/pack_num_bench.cpp b/ydb/library/yql/minikql/benchmark/pack_num/pack_num_bench.cpp
index d49aa682d0..76e4671e0e 100644
--- a/ydb/library/yql/minikql/benchmark/pack_num/pack_num_bench.cpp
+++ b/ydb/library/yql/minikql/benchmark/pack_num/pack_num_bench.cpp
@@ -1,226 +1,226 @@
#include <ydb/library/yql/minikql/pack_num.h>
-
+
#include <library/cpp/testing/benchmark/bench.h>
#include <library/cpp/packedtypes/longs.h>
-
-#include <dict/dictutil/pack.h>
-
-#include <util/generic/xrange.h>
-#include <util/generic/singleton.h>
-#include <util/random/random.h>
-
-namespace {
-
-template <ui32 UPPER, size_t MAX_BYTE_SIZE = (1 << 15) - (1 << 10)>
-struct TSamples32 {
- constexpr static size_t COUNT = MAX_BYTE_SIZE / sizeof(ui32);
- TSamples32() {
- for (size_t i: xrange(COUNT)) {
- Data[i] = RandomNumber<ui32>(UPPER);
- }
- }
-
- ui32 Data[COUNT];
-};
-
-template <ui64 UPPER, size_t MAX_BYTE_SIZE = (1 << 15) - (1 << 10)>
-struct TSamples64 {
- constexpr static size_t COUNT = MAX_BYTE_SIZE / sizeof(ui64);
- TSamples64() {
- for (size_t i: xrange(COUNT)) {
- Data[i] = RandomNumber<ui64>(UPPER);
- }
- }
-
- ui64 Data[COUNT];
-};
-
-template <ui32 UPPER, typename TCoder, size_t MAX_BYTE_SIZE = (1 << 15) - (1 << 10)>
-struct TCodedData32 {
- constexpr static size_t BYTES_PER_NUM = sizeof(ui32) + 2;
- constexpr static size_t COUNT = MAX_BYTE_SIZE / BYTES_PER_NUM;
- TCodedData32() {
- for (size_t i: xrange(COUNT)) {
- Length[i] = TCoder()(RandomNumber<ui32>(UPPER), Data + i * BYTES_PER_NUM);
- }
- }
-
- char Data[MAX_BYTE_SIZE + BYTES_PER_NUM];
- size_t Length[COUNT];
-};
-
-template <ui64 UPPER, typename TCoder, size_t MAX_BYTE_SIZE = (1 << 15) - (1 << 10)>
-struct TCodedData64 {
- constexpr static size_t BYTES_PER_NUM = sizeof(ui64) + 2;
- constexpr static size_t COUNT = MAX_BYTE_SIZE / BYTES_PER_NUM;
- TCodedData64() {
- for (size_t i: xrange(COUNT)) {
- Length[i] = TCoder()(RandomNumber<ui64>(UPPER), Data + i * BYTES_PER_NUM);
- }
- }
-
- char Data[MAX_BYTE_SIZE + BYTES_PER_NUM];
- size_t Length[COUNT];
-};
-
-struct TKikimrCoder32 {
- size_t operator() (ui32 num, char* buf) const {
- return NKikimr::Pack32(num, buf);
- }
-};
-
-struct TKikimrCoder64 {
- size_t operator() (ui64 num, char* buf) const {
- return NKikimr::Pack64(num, buf);
- }
-};
-
-struct TPackedTypesCoder32 {
- size_t operator() (ui32 num, char* buf) const {
- return Pack32(num, buf) - buf;
- }
-};
-
-struct TPackedTypesCoder64 {
- size_t operator() (ui64 num, char* buf) const {
- return Pack64(num, buf) - buf;
- }
-};
-
-struct TDictUtilsCoder32 {
- size_t operator() (ui32 num, char* buf) const {
- return PackU32(num, buf);
- }
-};
-
-struct TDictUtilsCoder64 {
- size_t operator() (ui64 num, char* buf) const {
- return PackU64(num, buf);
- }
-};
-
-} // unnamed
-
-#define DEF_WRITE_BENCH(base, limit) \
- Y_CPU_BENCHMARK(Write##base##_Kikimr_##limit, iface) { \
- char buffer[sizeof(ui##base) + 1]; \
- auto& data = Default<TSamples##base<limit>>(); \
- for (size_t i = 0; i < iface.Iterations(); ++i) { \
- Y_DO_NOT_OPTIMIZE_AWAY(NKikimr::Pack##base(data.Data[i % data.COUNT], buffer)); \
- NBench::Clobber(); \
- } \
- } \
- Y_CPU_BENCHMARK(Write##base##_PackedTypes_##limit, iface) { \
- char buffer[sizeof(ui##base) + 1]; \
- auto& data = Default<TSamples##base<limit>>(); \
- for (size_t i = 0; i < iface.Iterations(); ++i) { \
- Y_DO_NOT_OPTIMIZE_AWAY(Pack##base(data.Data[i % data.COUNT], buffer)); \
- NBench::Clobber(); \
- } \
- } \
- Y_CPU_BENCHMARK(Write##base##_DictUtils_##limit, iface) { \
- char buffer[sizeof(ui##base) + 1]; \
- auto& data = Default<TSamples##base<limit>>(); \
- for (size_t i = 0; i < iface.Iterations(); ++i) { \
- Y_DO_NOT_OPTIMIZE_AWAY(PackU##base(data.Data[i % data.COUNT], buffer)); \
- NBench::Clobber(); \
- } \
- }
-
-
-#define DEF_READ_BENCH(base, limit) \
- Y_CPU_BENCHMARK(Read##base##_KikimrLong_##limit, iface) { \
- ui##base num = 0; \
- NBench::Escape(&num); \
- const auto& data = *HugeSingleton<TCodedData##base<limit, TKikimrCoder##base>>(); \
- for (size_t i = 0; i < iface.Iterations(); ++i) { \
- const size_t pos = i % data.COUNT; \
- Y_DO_NOT_OPTIMIZE_AWAY(NKikimr::Unpack##base(data.Data + pos * data.BYTES_PER_NUM, data.BYTES_PER_NUM, num)); \
- NBench::Clobber(); \
- } \
- } \
- Y_CPU_BENCHMARK(Read##base##_KikimrShort_##limit, iface) { \
- ui##base num = 0; \
- NBench::Escape(&num); \
- const auto& data = *HugeSingleton<TCodedData##base<limit, TKikimrCoder##base>>(); \
- for (size_t i = 0; i < iface.Iterations(); ++i) { \
- const size_t pos = i % data.COUNT; \
- Y_DO_NOT_OPTIMIZE_AWAY(NKikimr::Unpack##base(data.Data + pos * data.BYTES_PER_NUM, data.Length[pos], num)); \
- NBench::Clobber(); \
- } \
- } \
- Y_CPU_BENCHMARK(Read##base##_PackedTypes_##limit, iface) { \
- ui##base num = 0; \
- NBench::Escape(&num); \
- const auto& data = *HugeSingleton<TCodedData##base<limit, TPackedTypesCoder##base>>(); \
- for (size_t i = 0; i < iface.Iterations(); ++i) { \
- const size_t pos = i % data.COUNT; \
- Y_DO_NOT_OPTIMIZE_AWAY(Unpack##base(num, data.Data + pos * data.BYTES_PER_NUM)); \
- NBench::Clobber(); \
- } \
- } \
- Y_CPU_BENCHMARK(Read##base##_DictUtils_##limit, iface) { \
- ui##base num = 0; \
- NBench::Escape(&num); \
- const auto& data = *HugeSingleton<TCodedData##base<limit, TDictUtilsCoder##base>>(); \
- for (size_t i = 0; i < iface.Iterations(); ++i) { \
- const size_t pos = i % data.COUNT; \
- Y_DO_NOT_OPTIMIZE_AWAY(UnpackU##base(&num, data.Data + pos * data.BYTES_PER_NUM)); \
- NBench::Clobber(); \
- } \
- }
-
-
-DEF_WRITE_BENCH(32, 10)
-DEF_WRITE_BENCH(32, 126)
-DEF_WRITE_BENCH(32, 127)
-DEF_WRITE_BENCH(32, 128)
-DEF_WRITE_BENCH(32, 254)
-DEF_WRITE_BENCH(32, 255)
-DEF_WRITE_BENCH(32, 256)
-DEF_WRITE_BENCH(32, 65534)
-DEF_WRITE_BENCH(32, 65535)
-DEF_WRITE_BENCH(32, 65536)
-DEF_WRITE_BENCH(32, 4294967295)
-
-DEF_WRITE_BENCH(64, 10)
-DEF_WRITE_BENCH(64, 126)
-DEF_WRITE_BENCH(64, 127)
-DEF_WRITE_BENCH(64, 128)
-DEF_WRITE_BENCH(64, 254)
-DEF_WRITE_BENCH(64, 255)
-DEF_WRITE_BENCH(64, 256)
-DEF_WRITE_BENCH(64, 65534)
-DEF_WRITE_BENCH(64, 65535)
-DEF_WRITE_BENCH(64, 65536)
-DEF_WRITE_BENCH(64, 4294967294ull)
-DEF_WRITE_BENCH(64, 4294967295ull)
-DEF_WRITE_BENCH(64, 4294967296ull)
-DEF_WRITE_BENCH(64, 18446744073709551615ull)
-
-DEF_READ_BENCH(32, 10)
-DEF_READ_BENCH(32, 126)
-DEF_READ_BENCH(32, 127)
-DEF_READ_BENCH(32, 128)
-DEF_READ_BENCH(32, 254)
-DEF_READ_BENCH(32, 255)
-DEF_READ_BENCH(32, 256)
-DEF_READ_BENCH(32, 65534)
-DEF_READ_BENCH(32, 65535)
-DEF_READ_BENCH(32, 65536)
-DEF_READ_BENCH(32, 4294967295)
-
-DEF_READ_BENCH(64, 10)
-DEF_READ_BENCH(64, 126)
-DEF_READ_BENCH(64, 127)
-DEF_READ_BENCH(64, 128)
-DEF_READ_BENCH(64, 254)
-DEF_READ_BENCH(64, 255)
-DEF_READ_BENCH(64, 256)
-DEF_READ_BENCH(64, 65534)
-DEF_READ_BENCH(64, 65535)
-DEF_READ_BENCH(64, 65536)
-DEF_READ_BENCH(64, 4294967294ull)
-DEF_READ_BENCH(64, 4294967295ull)
-DEF_READ_BENCH(64, 4294967296ull)
-DEF_READ_BENCH(64, 18446744073709551615ull)
+
+#include <dict/dictutil/pack.h>
+
+#include <util/generic/xrange.h>
+#include <util/generic/singleton.h>
+#include <util/random/random.h>
+
+namespace {
+
+template <ui32 UPPER, size_t MAX_BYTE_SIZE = (1 << 15) - (1 << 10)>
+struct TSamples32 {
+ constexpr static size_t COUNT = MAX_BYTE_SIZE / sizeof(ui32);
+ TSamples32() {
+ for (size_t i: xrange(COUNT)) {
+ Data[i] = RandomNumber<ui32>(UPPER);
+ }
+ }
+
+ ui32 Data[COUNT];
+};
+
+template <ui64 UPPER, size_t MAX_BYTE_SIZE = (1 << 15) - (1 << 10)>
+struct TSamples64 {
+ constexpr static size_t COUNT = MAX_BYTE_SIZE / sizeof(ui64);
+ TSamples64() {
+ for (size_t i: xrange(COUNT)) {
+ Data[i] = RandomNumber<ui64>(UPPER);
+ }
+ }
+
+ ui64 Data[COUNT];
+};
+
+template <ui32 UPPER, typename TCoder, size_t MAX_BYTE_SIZE = (1 << 15) - (1 << 10)>
+struct TCodedData32 {
+ constexpr static size_t BYTES_PER_NUM = sizeof(ui32) + 2;
+ constexpr static size_t COUNT = MAX_BYTE_SIZE / BYTES_PER_NUM;
+ TCodedData32() {
+ for (size_t i: xrange(COUNT)) {
+ Length[i] = TCoder()(RandomNumber<ui32>(UPPER), Data + i * BYTES_PER_NUM);
+ }
+ }
+
+ char Data[MAX_BYTE_SIZE + BYTES_PER_NUM];
+ size_t Length[COUNT];
+};
+
+template <ui64 UPPER, typename TCoder, size_t MAX_BYTE_SIZE = (1 << 15) - (1 << 10)>
+struct TCodedData64 {
+ constexpr static size_t BYTES_PER_NUM = sizeof(ui64) + 2;
+ constexpr static size_t COUNT = MAX_BYTE_SIZE / BYTES_PER_NUM;
+ TCodedData64() {
+ for (size_t i: xrange(COUNT)) {
+ Length[i] = TCoder()(RandomNumber<ui64>(UPPER), Data + i * BYTES_PER_NUM);
+ }
+ }
+
+ char Data[MAX_BYTE_SIZE + BYTES_PER_NUM];
+ size_t Length[COUNT];
+};
+
+struct TKikimrCoder32 {
+ size_t operator() (ui32 num, char* buf) const {
+ return NKikimr::Pack32(num, buf);
+ }
+};
+
+struct TKikimrCoder64 {
+ size_t operator() (ui64 num, char* buf) const {
+ return NKikimr::Pack64(num, buf);
+ }
+};
+
+struct TPackedTypesCoder32 {
+ size_t operator() (ui32 num, char* buf) const {
+ return Pack32(num, buf) - buf;
+ }
+};
+
+struct TPackedTypesCoder64 {
+ size_t operator() (ui64 num, char* buf) const {
+ return Pack64(num, buf) - buf;
+ }
+};
+
+struct TDictUtilsCoder32 {
+ size_t operator() (ui32 num, char* buf) const {
+ return PackU32(num, buf);
+ }
+};
+
+struct TDictUtilsCoder64 {
+ size_t operator() (ui64 num, char* buf) const {
+ return PackU64(num, buf);
+ }
+};
+
+} // unnamed
+
+#define DEF_WRITE_BENCH(base, limit) \
+ Y_CPU_BENCHMARK(Write##base##_Kikimr_##limit, iface) { \
+ char buffer[sizeof(ui##base) + 1]; \
+ auto& data = Default<TSamples##base<limit>>(); \
+ for (size_t i = 0; i < iface.Iterations(); ++i) { \
+ Y_DO_NOT_OPTIMIZE_AWAY(NKikimr::Pack##base(data.Data[i % data.COUNT], buffer)); \
+ NBench::Clobber(); \
+ } \
+ } \
+ Y_CPU_BENCHMARK(Write##base##_PackedTypes_##limit, iface) { \
+ char buffer[sizeof(ui##base) + 1]; \
+ auto& data = Default<TSamples##base<limit>>(); \
+ for (size_t i = 0; i < iface.Iterations(); ++i) { \
+ Y_DO_NOT_OPTIMIZE_AWAY(Pack##base(data.Data[i % data.COUNT], buffer)); \
+ NBench::Clobber(); \
+ } \
+ } \
+ Y_CPU_BENCHMARK(Write##base##_DictUtils_##limit, iface) { \
+ char buffer[sizeof(ui##base) + 1]; \
+ auto& data = Default<TSamples##base<limit>>(); \
+ for (size_t i = 0; i < iface.Iterations(); ++i) { \
+ Y_DO_NOT_OPTIMIZE_AWAY(PackU##base(data.Data[i % data.COUNT], buffer)); \
+ NBench::Clobber(); \
+ } \
+ }
+
+
+#define DEF_READ_BENCH(base, limit) \
+ Y_CPU_BENCHMARK(Read##base##_KikimrLong_##limit, iface) { \
+ ui##base num = 0; \
+ NBench::Escape(&num); \
+ const auto& data = *HugeSingleton<TCodedData##base<limit, TKikimrCoder##base>>(); \
+ for (size_t i = 0; i < iface.Iterations(); ++i) { \
+ const size_t pos = i % data.COUNT; \
+ Y_DO_NOT_OPTIMIZE_AWAY(NKikimr::Unpack##base(data.Data + pos * data.BYTES_PER_NUM, data.BYTES_PER_NUM, num)); \
+ NBench::Clobber(); \
+ } \
+ } \
+ Y_CPU_BENCHMARK(Read##base##_KikimrShort_##limit, iface) { \
+ ui##base num = 0; \
+ NBench::Escape(&num); \
+ const auto& data = *HugeSingleton<TCodedData##base<limit, TKikimrCoder##base>>(); \
+ for (size_t i = 0; i < iface.Iterations(); ++i) { \
+ const size_t pos = i % data.COUNT; \
+ Y_DO_NOT_OPTIMIZE_AWAY(NKikimr::Unpack##base(data.Data + pos * data.BYTES_PER_NUM, data.Length[pos], num)); \
+ NBench::Clobber(); \
+ } \
+ } \
+ Y_CPU_BENCHMARK(Read##base##_PackedTypes_##limit, iface) { \
+ ui##base num = 0; \
+ NBench::Escape(&num); \
+ const auto& data = *HugeSingleton<TCodedData##base<limit, TPackedTypesCoder##base>>(); \
+ for (size_t i = 0; i < iface.Iterations(); ++i) { \
+ const size_t pos = i % data.COUNT; \
+ Y_DO_NOT_OPTIMIZE_AWAY(Unpack##base(num, data.Data + pos * data.BYTES_PER_NUM)); \
+ NBench::Clobber(); \
+ } \
+ } \
+ Y_CPU_BENCHMARK(Read##base##_DictUtils_##limit, iface) { \
+ ui##base num = 0; \
+ NBench::Escape(&num); \
+ const auto& data = *HugeSingleton<TCodedData##base<limit, TDictUtilsCoder##base>>(); \
+ for (size_t i = 0; i < iface.Iterations(); ++i) { \
+ const size_t pos = i % data.COUNT; \
+ Y_DO_NOT_OPTIMIZE_AWAY(UnpackU##base(&num, data.Data + pos * data.BYTES_PER_NUM)); \
+ NBench::Clobber(); \
+ } \
+ }
+
+
+DEF_WRITE_BENCH(32, 10)
+DEF_WRITE_BENCH(32, 126)
+DEF_WRITE_BENCH(32, 127)
+DEF_WRITE_BENCH(32, 128)
+DEF_WRITE_BENCH(32, 254)
+DEF_WRITE_BENCH(32, 255)
+DEF_WRITE_BENCH(32, 256)
+DEF_WRITE_BENCH(32, 65534)
+DEF_WRITE_BENCH(32, 65535)
+DEF_WRITE_BENCH(32, 65536)
+DEF_WRITE_BENCH(32, 4294967295)
+
+DEF_WRITE_BENCH(64, 10)
+DEF_WRITE_BENCH(64, 126)
+DEF_WRITE_BENCH(64, 127)
+DEF_WRITE_BENCH(64, 128)
+DEF_WRITE_BENCH(64, 254)
+DEF_WRITE_BENCH(64, 255)
+DEF_WRITE_BENCH(64, 256)
+DEF_WRITE_BENCH(64, 65534)
+DEF_WRITE_BENCH(64, 65535)
+DEF_WRITE_BENCH(64, 65536)
+DEF_WRITE_BENCH(64, 4294967294ull)
+DEF_WRITE_BENCH(64, 4294967295ull)
+DEF_WRITE_BENCH(64, 4294967296ull)
+DEF_WRITE_BENCH(64, 18446744073709551615ull)
+
+DEF_READ_BENCH(32, 10)
+DEF_READ_BENCH(32, 126)
+DEF_READ_BENCH(32, 127)
+DEF_READ_BENCH(32, 128)
+DEF_READ_BENCH(32, 254)
+DEF_READ_BENCH(32, 255)
+DEF_READ_BENCH(32, 256)
+DEF_READ_BENCH(32, 65534)
+DEF_READ_BENCH(32, 65535)
+DEF_READ_BENCH(32, 65536)
+DEF_READ_BENCH(32, 4294967295)
+
+DEF_READ_BENCH(64, 10)
+DEF_READ_BENCH(64, 126)
+DEF_READ_BENCH(64, 127)
+DEF_READ_BENCH(64, 128)
+DEF_READ_BENCH(64, 254)
+DEF_READ_BENCH(64, 255)
+DEF_READ_BENCH(64, 256)
+DEF_READ_BENCH(64, 65534)
+DEF_READ_BENCH(64, 65535)
+DEF_READ_BENCH(64, 65536)
+DEF_READ_BENCH(64, 4294967294ull)
+DEF_READ_BENCH(64, 4294967295ull)
+DEF_READ_BENCH(64, 4294967296ull)
+DEF_READ_BENCH(64, 18446744073709551615ull)
diff --git a/ydb/library/yql/minikql/benchmark/pack_num/ya.make b/ydb/library/yql/minikql/benchmark/pack_num/ya.make
index c0c74da450..f9ea9ef17c 100644
--- a/ydb/library/yql/minikql/benchmark/pack_num/ya.make
+++ b/ydb/library/yql/minikql/benchmark/pack_num/ya.make
@@ -1,19 +1,19 @@
Y_BENCHMARK()
-
-ALLOCATOR(B)
-
-OWNER(
- udovichenko-r
-)
-
-PEERDIR(
+
+ALLOCATOR(B)
+
+OWNER(
+ udovichenko-r
+)
+
+PEERDIR(
ydb/library/yql/minikql
library/cpp/packedtypes
- dict/dictutil
-)
-
-SRCS(
- pack_num_bench.cpp
-)
-
-END()
+ dict/dictutil
+)
+
+SRCS(
+ pack_num_bench.cpp
+)
+
+END()
diff --git a/ydb/library/yql/minikql/benchmark/ya.make b/ydb/library/yql/minikql/benchmark/ya.make
index ad658474b0..58cbb9450d 100644
--- a/ydb/library/yql/minikql/benchmark/ya.make
+++ b/ydb/library/yql/minikql/benchmark/ya.make
@@ -1,10 +1,10 @@
-OWNER(
+OWNER(
g:kikimr
g:yql
- udovichenko-r
-)
-
-RECURSE(
- pack_num
- pack_num/metrics
-)
+ udovichenko-r
+)
+
+RECURSE(
+ pack_num
+ pack_num/metrics
+)
diff --git a/ydb/library/yql/minikql/codegen/codegen.cpp b/ydb/library/yql/minikql/codegen/codegen.cpp
index 6e4096af22..75782e7be0 100644
--- a/ydb/library/yql/minikql/codegen/codegen.cpp
+++ b/ydb/library/yql/minikql/codegen/codegen.cpp
@@ -16,7 +16,7 @@
#include <llvm/Support/SourceMgr.h>
#include <llvm/Support/TargetSelect.h>
#include <llvm/Support/Timer.h>
-#include <llvm/Support/ErrorHandling.h>
+#include <llvm/Support/ErrorHandling.h>
#include <llvm/Transforms/IPO.h>
#include <llvm/Transforms/IPO/PassManagerBuilder.h>
#include <llvm/Transforms/Instrumentation.h>
@@ -30,10 +30,10 @@
#include <util/generic/maybe.h>
#include <util/generic/singleton.h>
#include <util/generic/hash_set.h>
-#include <util/generic/hash.h>
+#include <util/generic/hash.h>
#include <util/generic/yexception.h>
-#include <util/generic/strbuf.h>
-#include <util/generic/string.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/string.h>
#include <util/stream/format.h>
#include <util/system/defaults.h>
#include <util/system/platform.h>
@@ -210,13 +210,13 @@ namespace NYql {
namespace NCodegen {
namespace {
-
- void FatalErrorHandler(void* user_data, const std::string& reason, bool gen_crash_diag) {
- Y_UNUSED(user_data);
- Y_UNUSED(gen_crash_diag);
- ythrow yexception() << "LLVM fatal error: " << reason.c_str();
- }
-
+
+ void FatalErrorHandler(void* user_data, const std::string& reason, bool gen_crash_diag) {
+ Y_UNUSED(user_data);
+ Y_UNUSED(gen_crash_diag);
+ ythrow yexception() << "LLVM fatal error: " << reason.c_str();
+ }
+
void AddAddressSanitizerPasses(const llvm::PassManagerBuilder& builder, llvm::legacy::PassManagerBase& pm) {
Y_UNUSED(builder);
pm.add(llvm::createAddressSanitizerFunctionPass());
@@ -239,7 +239,7 @@ namespace {
llvm::InitializeNativeTargetAsmPrinter();
llvm::InitializeNativeTargetAsmParser();
llvm::InitializeNativeTargetDisassembler();
- llvm::install_fatal_error_handler(&FatalErrorHandler, nullptr);
+ llvm::install_fatal_error_handler(&FatalErrorHandler, nullptr);
}
};
@@ -403,7 +403,7 @@ public:
}
void Compile(const TStringBuf compileOpts, TCompileStats* compileStats) override {
-
+
bool dumpTimers = compileOpts.Contains("time-passes");
bool disableOpt = compileOpts.Contains("disable-opt");
@@ -612,10 +612,10 @@ public:
}
}
- void LoadBitCode(TStringBuf bitcode, TStringBuf uniqId) override {
+ void LoadBitCode(TStringBuf bitcode, TStringBuf uniqId) override {
if (uniqId && LoadedModules_.contains(uniqId)) {
- return;
- }
+ return;
+ }
llvm::SMDiagnostic error;
auto buffer = llvm::MemoryBuffer::getMemBuffer(
llvm::StringRef(bitcode.data(), bitcode.size()));
@@ -630,25 +630,25 @@ public:
module->setTargetTriple(Triple_);
module->setDataLayout(Engine_->getDataLayout().getStringRepresentation());
- if (uniqId) {
+ if (uniqId) {
module->setModuleIdentifier(llvm::StringRef(uniqId.data(), uniqId.size()));
- }
+ }
if (llvm::Linker::linkModules(*Module_, std::move(module))) {
- TString err;
- err.append("LLVM: error linking module");
- if (uniqId) {
- err.append(' ').append(uniqId);
- }
- if (Diagnostic_.size()) {
- err.append(": ").append(Diagnostic_.c_str(), Diagnostic_.size());
- }
- ythrow yexception() << err;
- }
-
- if (uniqId) {
- LoadedModules_.emplace(uniqId);
- }
+ TString err;
+ err.append("LLVM: error linking module");
+ if (uniqId) {
+ err.append(' ').append(uniqId);
+ }
+ if (Diagnostic_.size()) {
+ err.append(": ").append(Diagnostic_.c_str(), Diagnostic_.size());
+ }
+ ythrow yexception() << err;
+ }
+
+ if (uniqId) {
+ LoadedModules_.emplace(uniqId);
+ }
}
void AddGlobalMapping(TStringBuf name, const void* address) override {
@@ -737,7 +737,7 @@ private:
THashMap<const void*, TString> ReverseGlobalMapping_;
TMaybe<TPatterns> Patterns_;
TTlsManager TlsManager_;
- THashSet<TString> LoadedModules_;
+ THashSet<TString> LoadedModules_;
};
ICodegen::TPtr
diff --git a/ydb/library/yql/minikql/codegen/codegen.h b/ydb/library/yql/minikql/codegen/codegen.h
index 41326b6e0f..74a37fad68 100644
--- a/ydb/library/yql/minikql/codegen/codegen.h
+++ b/ydb/library/yql/minikql/codegen/codegen.h
@@ -56,7 +56,7 @@ public:
virtual void* GetPointerToFunction(llvm::Function* function) = 0;
virtual ui64 GetFunctionCodeSize(llvm::Function* function) = 0;
virtual void ShowGeneratedFunctions(IOutputStream* out) = 0;
- virtual void LoadBitCode(TStringBuf bitcode, TStringBuf uniqId) = 0;
+ virtual void LoadBitCode(TStringBuf bitcode, TStringBuf uniqId) = 0;
virtual void AddGlobalMapping(TStringBuf name, const void* address) = 0;
virtual void TogglePerfJITEventListener() = 0;
diff --git a/ydb/library/yql/minikql/codegen/codegen_ut.cpp b/ydb/library/yql/minikql/codegen/codegen_ut.cpp
index 332e3fdcdc..9815540827 100644
--- a/ydb/library/yql/minikql/codegen/codegen_ut.cpp
+++ b/ydb/library/yql/minikql/codegen/codegen_ut.cpp
@@ -275,7 +275,7 @@ Y_UNIT_TEST_SUITE(TCodegenTests) {
Y_UNIT_TEST(FibFromBitCode) {
auto codegen = ICodegen::Make(ETarget::Native);
auto bitcode = NResource::Find("/llvm_bc/Funcs");
- codegen->LoadBitCode(bitcode, "Funcs");
+ codegen->LoadBitCode(bitcode, "Funcs");
auto func = codegen->GetModule().getFunction("fib");
codegen->Verify();
codegen->ExportSymbol(func);
@@ -288,7 +288,7 @@ Y_UNIT_TEST_SUITE(TCodegenTests) {
Y_UNIT_TEST(LinkWithNativeFunction) {
auto codegen = ICodegen::Make(ETarget::Native);
auto bitcode = NResource::Find("/llvm_bc/Funcs");
- codegen->LoadBitCode(bitcode, "Funcs");
+ codegen->LoadBitCode(bitcode, "Funcs");
auto func = codegen->GetModule().getFunction("sum_sqr");
codegen->AddGlobalMapping("mul", (void*)&sum);
codegen->ExportSymbol(func);
@@ -303,7 +303,7 @@ Y_UNIT_TEST_SUITE(TCodegenTests) {
auto codegen = ICodegen::Make(ETarget::Native);
auto mulFunc = CreateMulFunction(codegen->GetModule(), codegen->GetContext());
auto bitcode = NResource::Find("/llvm_bc/Funcs");
- codegen->LoadBitCode(bitcode, "Funcs");
+ codegen->LoadBitCode(bitcode, "Funcs");
auto func = codegen->GetModule().getFunction("sum_sqr");
codegen->ExportSymbol(func);
codegen->Verify();
@@ -316,7 +316,7 @@ Y_UNIT_TEST_SUITE(TCodegenTests) {
Y_UNIT_TEST(ReuseExternalCode) {
auto codegen = ICodegen::Make(ETarget::Native);
auto bitcode = NResource::Find("/llvm_bc/Funcs");
- codegen->LoadBitCode(bitcode, "Funcs");
+ codegen->LoadBitCode(bitcode, "Funcs");
auto func = codegen->GetModule().getFunction("sum_sqr2");
codegen->ExportSymbol(func);
codegen->Verify();
@@ -329,7 +329,7 @@ Y_UNIT_TEST_SUITE(TCodegenTests) {
Y_UNIT_TEST(UseObjectReference) {
auto codegen = ICodegen::Make(ETarget::Native);
auto bitcode = NResource::Find("/llvm_bc/Funcs");
- codegen->LoadBitCode(bitcode, "Funcs");
+ codegen->LoadBitCode(bitcode, "Funcs");
auto func = codegen->GetModule().getFunction("str_size");
codegen->ExportSymbol(func);
codegen->Verify();
@@ -355,7 +355,7 @@ Y_UNIT_TEST_SUITE(TCodegenTests) {
Y_UNIT_TEST(UseExternalFromGeneratedFunction) {
auto codegen = ICodegen::Make(ETarget::Native);
auto bitcode = NResource::Find("/llvm_bc/Funcs");
- codegen->LoadBitCode(bitcode, "Funcs");
+ codegen->LoadBitCode(bitcode, "Funcs");
auto func = CreateUseExternalFromGeneratedFunction(codegen->GetModule(), codegen->GetContext());
codegen->ExportSymbol(func);
codegen->AddGlobalMapping("mul", (void*)&mul);
@@ -369,7 +369,7 @@ Y_UNIT_TEST_SUITE(TCodegenTests) {
Y_UNIT_TEST(UseExternalFromGeneratedFunction_128bit_Compiled) {
auto codegen = ICodegen::Make(ETarget::Native);
auto bitcode = NResource::Find("/llvm_bc/Funcs");
- codegen->LoadBitCode(bitcode, "Funcs");
+ codegen->LoadBitCode(bitcode, "Funcs");
auto func = CreateUseExternalFromGeneratedFunction128(codegen, false);
codegen->ExportSymbol(func);
codegen->Verify();
@@ -395,7 +395,7 @@ Y_UNIT_TEST_SUITE(TCodegenTests) {
Y_UNIT_TEST(UseExternalFromGeneratedFunction_128bit_Bitcode) {
auto codegen = ICodegen::Make(ETarget::Native);
auto bitcode = NResource::Find("/llvm_bc/Funcs");
- codegen->LoadBitCode(bitcode, "Funcs");
+ codegen->LoadBitCode(bitcode, "Funcs");
auto func = CreateUseExternalFromGeneratedFunction128(codegen, true);
codegen->ExportSymbol(func);
codegen->Verify();
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp
index a613c9dcd9..58080aa3ef 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_apply.cpp
@@ -64,20 +64,20 @@ public:
}
}
- if (const auto codegen = dynamic_cast<ICodegeneratorRunNode*>(CallableNode)) {
- codegen->CreateRun(ctx, block, pointer, args);
- } else {
- const auto callable = GetNodeValue(CallableNode, ctx, block);
+ if (const auto codegen = dynamic_cast<ICodegeneratorRunNode*>(CallableNode)) {
+ codegen->CreateRun(ctx, block, pointer, args);
+ } else {
+ const auto callable = GetNodeValue(CallableNode, ctx, block);
const auto calleePtr = GetElementPtrInst::CreateInBounds(ctx.Ctx, {ConstantInt::get(idxType, 0), ConstantInt::get(idxType, 6)}, "callee_ptr", block);
const auto previous = new LoadInst(calleePtr, "previous", block);
const auto callee = CastInst::Create(Instruction::IntToPtr, ConstantInt::get(Type::getInt64Ty(context), ui64(&Position)), previous->getType(), "callee", block);
new StoreInst(callee, calleePtr, block);
- CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Run>(pointer, callable, ctx.Codegen, block, ctx.GetBuilder(), args);
+ CallBoxedValueVirtualMethod<NUdf::TBoxedValueAccessor::EMethod::Run>(pointer, callable, ctx.Codegen, block, ctx.GetBuilder(), args);
new StoreInst(previous, calleePtr, block);
- if (CallableNode->IsTemporaryValue()) {
- CleanupBoxed(callable, ctx, block);
- }
- }
+ if (CallableNode->IsTemporaryValue()) {
+ CleanupBoxed(callable, ctx, block);
+ }
+ }
for (const auto& arg : argsv) {
ValueUnRef(arg.second, arg.first, ctx, block);
}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp
index 4e13f63a14..4367835a8e 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_combine.cpp
@@ -1,13 +1,13 @@
-#include "mkql_combine.h"
-
+#include "mkql_combine.h"
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_stats_registry.h>
#include <ydb/library/yql/minikql/defs.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
+
+namespace NKikimr {
+namespace NMiniKQL {
+
TStatKey Combine_FlushesCount("Combine_FlushesCount", true);
TStatKey Combine_MaxRowsCount("Combine_MaxRowsCount", false);
@@ -545,19 +545,19 @@ class TCombineCoreWrapper: public TCustomValueCodegeneratorNode<TCombineCoreWrap
#ifndef MKQL_DISABLE_CODEGEN
using TCodegenValue = std::conditional_t<IsMultiRowState, TStreamCodegenSelfStatePlusValue<TState>, TStreamCodegenSelfStateValue<TState>>;
#endif
-public:
+public:
class TStreamValue : public TState {
- public:
+ public:
TStreamValue(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& stream, const TCombineCoreNodes& nodes, ui64 memLimit, TComputationContext& compCtx, const THashFunc& hash, const TEqualsFunc& equal)
: TState(memInfo, hash, equal)
- , Stream(std::move(stream))
+ , Stream(std::move(stream))
, Nodes(nodes)
, MemLimit(memLimit)
, CompCtx(compCtx)
{}
-
- private:
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
+
+ private:
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
for (;;) {
if (IsMultiRowState && Iterator) {
if constexpr (StateContainerOpt) {
@@ -571,8 +571,8 @@ public:
return NUdf::EFetchStatus::Ok;
}
Iterator.Clear();
- }
-
+ }
+
if (IsEmpty()) {
switch (InputStatus) {
case NUdf::EFetchStatus::Ok: break;
@@ -581,8 +581,8 @@ public:
case NUdf::EFetchStatus::Yield:
InputStatus = NUdf::EFetchStatus::Ok;
return NUdf::EFetchStatus::Yield;
- }
-
+ }
+
const auto initUsage = MemLimit ? CompCtx.HolderFactory.GetMemoryUsed() : 0ULL;
do {
@@ -597,8 +597,8 @@ public:
} while (!CompCtx.template CheckAdjustedMemLimit<TrackRss>(MemLimit, initUsage));
PushStat(CompCtx.Stats);
- }
-
+ }
+
if (NUdf::TUnboxedValue key, state; Extract(key, state)) {
NUdf::TUnboxedValue finishItem = Nodes.FinishItem(CompCtx, key, state);
@@ -608,17 +608,17 @@ public:
result = finishItem.Release().GetOptionalValueIf<StateContainerOpt>();
return NUdf::EFetchStatus::Ok;
}
- }
- }
- }
-
+ }
+ }
+ }
+
const NUdf::TUnboxedValue Stream;
NUdf::TUnboxedValue Iterator;
const TCombineCoreNodes Nodes;
const ui64 MemLimit;
TComputationContext& CompCtx;
- };
-
+ };
+
TCombineCoreWrapper(TComputationMutables& mutables, IComputationNode* stream, const TCombineCoreNodes& nodes, TKeyTypes&& keyTypes, bool isTuple, ui64 memLimit)
: TBaseComputation(mutables)
, Stream(stream)
@@ -936,32 +936,32 @@ private:
}
-IComputationNode* WrapCombineCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+IComputationNode* WrapCombineCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
MKQL_ENSURE(callable.GetInputsCount() == 9U, "Expected 9 args");
-
+
const auto type = callable.GetType()->GetReturnType();
const auto finishResultType = callable.GetInput(7).GetStaticType();
- MKQL_ENSURE(finishResultType->IsList() || finishResultType->IsOptional() || finishResultType->IsStream(), "Expected list, stream or optional");
-
+ MKQL_ENSURE(finishResultType->IsList() || finishResultType->IsOptional() || finishResultType->IsStream(), "Expected list, stream or optional");
+
const auto keyType = callable.GetInput(2).GetStaticType();
- TKeyTypes keyTypes;
+ TKeyTypes keyTypes;
bool isTuple;
bool encoded;
GetDictionaryKeyTypes(keyType, keyTypes, isTuple, encoded);
Y_VERIFY(!encoded, "TODO");
const auto memLimit = AS_VALUE(TDataLiteral, callable.GetInput(8))->AsValue().Get<ui64>();
const bool trackRss = EGraphPerProcess::Single == ctx.GraphPerProcess;
-
+
const auto stream = LocateNode(ctx.NodeLocator, callable, 0);
const auto keyExtractorResultNode = LocateNode(ctx.NodeLocator, callable, 2);
const auto initResultNode = LocateNode(ctx.NodeLocator, callable, 4);
const auto updateResultNode = LocateNode(ctx.NodeLocator, callable, 6);
const auto finishResultNode = LocateNode(ctx.NodeLocator, callable, 7);
-
+
const auto itemNode = LocateExternalNode(ctx.NodeLocator, callable, 1);
const auto keyNode = LocateExternalNode(ctx.NodeLocator, callable, 3);
const auto stateNode = LocateExternalNode(ctx.NodeLocator, callable, 5);
-
+
const TCombineCoreNodes nodes = {
itemNode,
keyNode,
@@ -1021,10 +1021,10 @@ IComputationNode* WrapCombineCore(TCallable& callable, const TComputationNodeFac
return new TCombineCoreWrapper<false, false, false>(ctx.Mutables, stream, nodes, std::move(keyTypes), isTuple, memLimit);
}
}
- }
+ }
THROW yexception() << "Expected flow or stream.";
-}
-
-}
-}
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_combine.h b/ydb/library/yql/minikql/comp_nodes/mkql_combine.h
index 938ba8a650..5b97d3e3de 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_combine.h
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_combine.h
@@ -1,11 +1,11 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-IComputationNode* WrapCombineCore(TCallable& callable, const TComputationNodeFactoryContext& ctx);
-
-}
-}
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+IComputationNode* WrapCombineCore(TCallable& callable, const TComputationNodeFactoryContext& ctx);
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_factories.h b/ydb/library/yql/minikql/comp_nodes/mkql_factories.h
index 51ca6fdf2c..51e00313ee 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_factories.h
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_factories.h
@@ -6,7 +6,7 @@ namespace NMiniKQL {
TComputationNodeFactory GetBuiltinFactory();
-TComputationNodeFactory GetCompositeWithBuiltinFactory(TVector<TComputationNodeFactory> factories);
-
+TComputationNodeFactory GetCompositeWithBuiltinFactory(TVector<TComputationNodeFactory> factories);
+
}
}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
index 57599acb64..92643cc47a 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_factory.cpp
@@ -12,7 +12,7 @@
#include "mkql_chopper.h"
#include "mkql_coalesce.h"
#include "mkql_collect.h"
-#include "mkql_combine.h"
+#include "mkql_combine.h"
#include "mkql_contains.h"
#include "mkql_decimal_div.h"
#include "mkql_decimal_mod.h"
@@ -33,7 +33,7 @@
#include "mkql_fromstring.h"
#include "mkql_fromyson.h"
#include "mkql_guess.h"
-#include "mkql_group.h"
+#include "mkql_group.h"
#include "mkql_heap.h"
#include "mkql_hasitems.h"
#include "mkql_hopping.h"
@@ -84,7 +84,7 @@
#include "mkql_tostring.h"
#include "mkql_udf.h"
#include "mkql_unwrap.h"
-#include "mkql_varitem.h"
+#include "mkql_varitem.h"
#include "mkql_visitall.h"
#include "mkql_way.h"
#include "mkql_weakmember.h"
@@ -316,17 +316,17 @@ TComputationNodeFactory GetBuiltinFactory() {
};
}
-TComputationNodeFactory GetCompositeWithBuiltinFactory(TVector<TComputationNodeFactory> factories) {
- return [factories = std::move(factories), builtins = GetBuiltinFactory()](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
- for (auto& f: factories) {
- if (auto res = f(callable, ctx)) {
- return res;
- }
- }
-
- return builtins(callable, ctx);
- };
-}
-
+TComputationNodeFactory GetCompositeWithBuiltinFactory(TVector<TComputationNodeFactory> factories) {
+ return [factories = std::move(factories), builtins = GetBuiltinFactory()](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+ for (auto& f: factories) {
+ if (auto res = f(callable, ctx)) {
+ return res;
+ }
+ }
+
+ return builtins(callable, ctx);
+ };
}
+
}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp
index a9d49e109d..08612a2958 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_flatmap.cpp
@@ -1104,9 +1104,9 @@ private:
current = IsNewStream ? NewItem->GetValue(CompCtx) : NewItem->GetValue(CompCtx).GetListIterator();
}
- auto status = NUdf::EFetchStatus::Ok;
+ auto status = NUdf::EFetchStatus::Ok;
if constexpr (IsNewStream) {
- status = current.Fetch(result);
+ status = current.Fetch(result);
if (NUdf::EFetchStatus::Finish == status) {
continue;
}
@@ -1117,7 +1117,7 @@ private:
}
Current = std::move(current);
- return status;
+ return status;
}
}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp
index 0d0ff62fae..bf14b2bc9c 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_group.cpp
@@ -1,243 +1,243 @@
-#include "mkql_group.h"
-
+#include "mkql_group.h"
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_stats_registry.h>
#include <ydb/library/yql/minikql/compact_hash.h>
#include <ydb/library/yql/minikql/defs.h>
-
-#include <util/generic/maybe.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
+
+#include <util/generic/maybe.h>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
namespace {
-template <bool WithHandler>
-class TGroupingCoreWrapper: public TMutableComputationNode<TGroupingCoreWrapper<WithHandler>> {
- using TSelf = TGroupingCoreWrapper;
- typedef TMutableComputationNode<TSelf> TBaseComputation;
-public:
-
- class TSplitStreamValue : public TComputationValue<TSplitStreamValue> {
- public:
- using TBase = TComputationValue<TSplitStreamValue>;
-
- enum EState {
- AtStart,
- AtGroupStart,
- Fetching,
- GroupFinished,
- Finished,
- };
-
+template <bool WithHandler>
+class TGroupingCoreWrapper: public TMutableComputationNode<TGroupingCoreWrapper<WithHandler>> {
+ using TSelf = TGroupingCoreWrapper;
+ typedef TMutableComputationNode<TSelf> TBaseComputation;
+public:
+
+ class TSplitStreamValue : public TComputationValue<TSplitStreamValue> {
+ public:
+ using TBase = TComputationValue<TSplitStreamValue>;
+
+ enum EState {
+ AtStart,
+ AtGroupStart,
+ Fetching,
+ GroupFinished,
+ Finished,
+ };
+
TSplitStreamValue(TMemoryUsageInfo* memInfo, TComputationContext& compCtx, const TSelf* self, NUdf::TUnboxedValue&& stream)
- : TBase(memInfo)
+ : TBase(memInfo)
, CompCtx(compCtx)
- , Self(self)
- , Stream(std::move(stream))
- {
- }
-
- NUdf::EFetchStatus NextKey(NUdf::TUnboxedValue& key) {
- if (Fetching == State || AtGroupStart == State) {
- NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
- for (NUdf::TUnboxedValue item; NUdf::EFetchStatus::Ok == status; status = Fetch(item)) {
- }
- if (NUdf::EFetchStatus::Finish != status) {
- return status;
- }
- }
-
- if (Finished == State) {
- return NUdf::EFetchStatus::Finish;
- }
-
- if (GroupFinished != State) {
- auto status = Stream.Fetch(Value);
- if (NUdf::EFetchStatus::Finish == status) {
- State = Finished;
- }
- if (NUdf::EFetchStatus::Ok != status) {
- return status;
- }
- }
-
+ , Self(self)
+ , Stream(std::move(stream))
+ {
+ }
+
+ NUdf::EFetchStatus NextKey(NUdf::TUnboxedValue& key) {
+ if (Fetching == State || AtGroupStart == State) {
+ NUdf::EFetchStatus status = NUdf::EFetchStatus::Ok;
+ for (NUdf::TUnboxedValue item; NUdf::EFetchStatus::Ok == status; status = Fetch(item)) {
+ }
+ if (NUdf::EFetchStatus::Finish != status) {
+ return status;
+ }
+ }
+
+ if (Finished == State) {
+ return NUdf::EFetchStatus::Finish;
+ }
+
+ if (GroupFinished != State) {
+ auto status = Stream.Fetch(Value);
+ if (NUdf::EFetchStatus::Finish == status) {
+ State = Finished;
+ }
+ if (NUdf::EFetchStatus::Ok != status) {
+ return status;
+ }
+ }
+
Self->KeyExtractorItemNode->SetValue(CompCtx, NUdf::TUnboxedValue(Value));
key = Self->KeyExtractorResultNode->GetValue(CompCtx);
Self->GroupSwitchKeyNode->SetValue(CompCtx, NUdf::TUnboxedValue(key));
Self->GroupSwitchItemNode->SetValue(CompCtx, NUdf::TUnboxedValue(Value));
- State = AtGroupStart;
-
- return NUdf::EFetchStatus::Ok;
- }
-
- private:
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
- if (Finished == State) {
- return NUdf::EFetchStatus::Finish;
- }
-
- if (AtGroupStart != State) {
- auto status = Stream.Fetch(Value);
- if (NUdf::EFetchStatus::Finish == status) {
- State = Finished;
- }
- if (NUdf::EFetchStatus::Ok != status) {
- return status;
- }
- }
-
- if (Fetching == State) {
+ State = AtGroupStart;
+
+ return NUdf::EFetchStatus::Ok;
+ }
+
+ private:
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
+ if (Finished == State) {
+ return NUdf::EFetchStatus::Finish;
+ }
+
+ if (AtGroupStart != State) {
+ auto status = Stream.Fetch(Value);
+ if (NUdf::EFetchStatus::Finish == status) {
+ State = Finished;
+ }
+ if (NUdf::EFetchStatus::Ok != status) {
+ return status;
+ }
+ }
+
+ if (Fetching == State) {
Self->GroupSwitchItemNode->SetValue(CompCtx, NUdf::TUnboxedValue(Value));
- if (Self->GroupSwitchResultNode->GetValue(CompCtx).template Get<bool>()) {
- State = GroupFinished;
- return NUdf::EFetchStatus::Finish;
- }
- } else {
- State = Fetching;
- }
-
- if constexpr (WithHandler) {
- Self->HandlerItemNode->SetValue(CompCtx, std::move(Value));
- result = Self->HandlerResultNode->GetValue(CompCtx);
- } else {
- result = std::move(Value);
- }
- return NUdf::EFetchStatus::Ok;
- }
-
- private:
+ if (Self->GroupSwitchResultNode->GetValue(CompCtx).template Get<bool>()) {
+ State = GroupFinished;
+ return NUdf::EFetchStatus::Finish;
+ }
+ } else {
+ State = Fetching;
+ }
+
+ if constexpr (WithHandler) {
+ Self->HandlerItemNode->SetValue(CompCtx, std::move(Value));
+ result = Self->HandlerResultNode->GetValue(CompCtx);
+ } else {
+ result = std::move(Value);
+ }
+ return NUdf::EFetchStatus::Ok;
+ }
+
+ private:
TComputationContext& CompCtx;
- const TSelf* const Self;
- NUdf::TUnboxedValue Stream;
- EState State = AtStart;
- NUdf::TUnboxedValue Value;
- };
-
- class TGroupStreamValue : public TComputationValue<TGroupStreamValue> {
- public:
- using TBase = TComputationValue<TGroupStreamValue>;
-
+ const TSelf* const Self;
+ NUdf::TUnboxedValue Stream;
+ EState State = AtStart;
+ NUdf::TUnboxedValue Value;
+ };
+
+ class TGroupStreamValue : public TComputationValue<TGroupStreamValue> {
+ public:
+ using TBase = TComputationValue<TGroupStreamValue>;
+
TGroupStreamValue(TMemoryUsageInfo* memInfo, TComputationContext& compCtx, const TSelf* self, NUdf::TUnboxedValue&& stream)
- : TBase(memInfo)
+ : TBase(memInfo)
, CompCtx(compCtx)
, SplitStream(CompCtx.HolderFactory.Create<TSplitStreamValue>(CompCtx, self, std::move(stream)))
- , SplitStreamValue(static_cast<TSplitStreamValue*>(SplitStream.AsBoxed().Get()))
- {
- }
-
- private:
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
- NUdf::TUnboxedValue key;
- auto status = SplitStreamValue->NextKey(key);
- if (status != NUdf::EFetchStatus::Ok) {
- return status;
- }
-
- NKikimr::NUdf::TUnboxedValue* itemsPtr;
+ , SplitStreamValue(static_cast<TSplitStreamValue*>(SplitStream.AsBoxed().Get()))
+ {
+ }
+
+ private:
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
+ NUdf::TUnboxedValue key;
+ auto status = SplitStreamValue->NextKey(key);
+ if (status != NUdf::EFetchStatus::Ok) {
+ return status;
+ }
+
+ NKikimr::NUdf::TUnboxedValue* itemsPtr;
result = CompCtx.HolderFactory.CreateDirectArrayHolder(2, itemsPtr);
- itemsPtr[0] = std::move(key);
- itemsPtr[1] = SplitStream;
-
- return status;
- }
-
- private:
+ itemsPtr[0] = std::move(key);
+ itemsPtr[1] = SplitStream;
+
+ return status;
+ }
+
+ private:
TComputationContext& CompCtx;
- NUdf::TUnboxedValue SplitStream;
- TSplitStreamValue* SplitStreamValue;
- };
-
- TGroupingCoreWrapper(TComputationMutables& mutables,
- IComputationNode* stream,
+ NUdf::TUnboxedValue SplitStream;
+ TSplitStreamValue* SplitStreamValue;
+ };
+
+ TGroupingCoreWrapper(TComputationMutables& mutables,
+ IComputationNode* stream,
IComputationExternalNode* keyExtractorItem,
IComputationNode* keyExtractorResult,
IComputationExternalNode* groupSwitchKey,
IComputationExternalNode* groupSwitchItem,
- IComputationNode* groupSwitchResult,
- IComputationExternalNode* handlerItem,
- IComputationNode* handlerResult)
- : TBaseComputation(mutables)
- , Stream(stream)
+ IComputationNode* groupSwitchResult,
+ IComputationExternalNode* handlerItem,
+ IComputationNode* handlerResult)
+ : TBaseComputation(mutables)
+ , Stream(stream)
, KeyExtractorItemNode(keyExtractorItem)
, KeyExtractorResultNode(keyExtractorResult)
, GroupSwitchKeyNode(groupSwitchKey)
, GroupSwitchItemNode(groupSwitchItem)
- , GroupSwitchResultNode(groupSwitchResult)
- , HandlerItemNode(handlerItem)
- , HandlerResultNode(handlerResult)
- {
- }
-
- NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
+ , GroupSwitchResultNode(groupSwitchResult)
+ , HandlerItemNode(handlerItem)
+ , HandlerResultNode(handlerResult)
+ {
+ }
+
+ NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
return ctx.HolderFactory.Create<TGroupStreamValue>(ctx, this, Stream->GetValue(ctx));
- }
-
-private:
- void RegisterDependencies() const final {
- this->DependsOn(Stream);
+ }
+
+private:
+ void RegisterDependencies() const final {
+ this->DependsOn(Stream);
this->DependsOn(KeyExtractorResultNode);
- this->DependsOn(GroupSwitchResultNode);
- this->DependsOn(HandlerResultNode);
+ this->DependsOn(GroupSwitchResultNode);
+ this->DependsOn(HandlerResultNode);
this->Own(KeyExtractorItemNode);
this->Own(GroupSwitchKeyNode);
this->Own(GroupSwitchItemNode);
- this->Own(HandlerItemNode);
- }
-
-private:
- IComputationNode* const Stream;
-
+ this->Own(HandlerItemNode);
+ }
+
+private:
+ IComputationNode* const Stream;
+
IComputationExternalNode* const KeyExtractorItemNode;
IComputationNode* const KeyExtractorResultNode;
-
+
IComputationExternalNode* const GroupSwitchKeyNode;
IComputationExternalNode* const GroupSwitchItemNode;
- IComputationNode* const GroupSwitchResultNode;
-
- IComputationExternalNode* const HandlerItemNode;
- IComputationNode* const HandlerResultNode;
-};
-
+ IComputationNode* const GroupSwitchResultNode;
+
+ IComputationExternalNode* const HandlerItemNode;
+ IComputationNode* const HandlerResultNode;
+};
+
}
-IComputationNode* WrapGroupingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- MKQL_ENSURE(callable.GetInputsCount() == 6 || callable.GetInputsCount() == 8, "Expected 6 or 8 args");
-
+IComputationNode* WrapGroupingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ MKQL_ENSURE(callable.GetInputsCount() == 6 || callable.GetInputsCount() == 8, "Expected 6 or 8 args");
+
const auto stream = LocateNode(ctx.NodeLocator, callable, 0);
const auto keyExtractorResult = LocateNode(ctx.NodeLocator, callable, 1);
const auto groupSwitchResult = LocateNode(ctx.NodeLocator, callable, 2);
const auto keyExtractorItem = LocateExternalNode(ctx.NodeLocator, callable, 3);
const auto groupSwitchKey = LocateExternalNode(ctx.NodeLocator, callable, 4);
const auto groupSwitchItem = LocateExternalNode(ctx.NodeLocator, callable, 5);
-
- if (callable.GetInputsCount() == 8) {
- auto handlerResult = LocateNode(ctx.NodeLocator, callable, 6);
- auto handlerItem = LocateExternalNode(ctx.NodeLocator, callable, 7);
- return new TGroupingCoreWrapper<true>(
- ctx.Mutables,
- stream,
- keyExtractorItem,
- keyExtractorResult,
- groupSwitchKey,
- groupSwitchItem,
- groupSwitchResult,
- handlerItem,
- handlerResult);
- }
-
- return new TGroupingCoreWrapper<false>(
+
+ if (callable.GetInputsCount() == 8) {
+ auto handlerResult = LocateNode(ctx.NodeLocator, callable, 6);
+ auto handlerItem = LocateExternalNode(ctx.NodeLocator, callable, 7);
+ return new TGroupingCoreWrapper<true>(
+ ctx.Mutables,
+ stream,
+ keyExtractorItem,
+ keyExtractorResult,
+ groupSwitchKey,
+ groupSwitchItem,
+ groupSwitchResult,
+ handlerItem,
+ handlerResult);
+ }
+
+ return new TGroupingCoreWrapper<false>(
ctx.Mutables,
stream,
keyExtractorItem,
keyExtractorResult,
groupSwitchKey,
groupSwitchItem,
- groupSwitchResult,
- nullptr,
- nullptr);
-}
-
-}
-}
+ groupSwitchResult,
+ nullptr,
+ nullptr);
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_group.h b/ydb/library/yql/minikql/comp_nodes/mkql_group.h
index 85c72e2cb4..bf46f8c700 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_group.h
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_group.h
@@ -1,11 +1,11 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-IComputationNode* WrapGroupingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx);
-
-}
-}
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+IComputationNode* WrapGroupingCore(TCallable& callable, const TComputationNodeFactoryContext& ctx);
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp
index 8eccbb27b5..0d80181ab0 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_iterator.cpp
@@ -57,7 +57,7 @@ private:
class TForwardListWrapper : public TMutableCodegeneratorNode<TForwardListWrapper> {
typedef TMutableCodegeneratorNode<TForwardListWrapper> TBaseComputation;
-public:
+public:
TForwardListWrapper(TComputationMutables& mutables, IComputationNode* stream)
: TBaseComputation(mutables, EValueRepresentation::Boxed), Stream(stream)
{}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp
index d2182bbc23..6acc41011a 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_join.cpp
@@ -1,15 +1,15 @@
#include "mkql_join.h"
-
+
#include <ydb/library/yql/minikql/computation/mkql_custom_list.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_pack.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_program_builder.h>
-
+
#include <util/system/tempfile.h>
#include <util/stream/file.h>
#include <util/system/fstat.h>
-#include <util/generic/ylimits.h>
+#include <util/generic/ylimits.h>
namespace NKikimr {
namespace NMiniKQL {
@@ -1396,7 +1396,7 @@ public:
return *Ctx;
}
- bool Spill() {
+ bool Spill() {
if (FileState) {
return false;
}
@@ -1418,7 +1418,7 @@ public:
LiveValue = std::move(liveValue);
}
- void Add(NUdf::TUnboxedValue&& value) {
+ void Add(NUdf::TUnboxedValue&& value) {
#ifndef NDEBUG
Y_VERIFY_DEBUG(!IsSealed);
#endif
@@ -1647,7 +1647,7 @@ public:
if (status == NUdf::EFetchStatus::Yield) {
return status;
}
-
+
if (status == NUdf::EFetchStatus::Finish) {
EatInput = false;
} else {
@@ -1989,7 +1989,7 @@ public:
, RequiredColumns(std::move(requiredColumns))
, LeftOutputColumns(std::move(leftOutputColumns))
, RightOutputColumns(std::move(rightOutputColumns))
- , MemLimit(memLimit)
+ , MemLimit(memLimit)
, SortedTableOrder(sortedTableOrder)
, KeyColumns(std::move(keyColumns))
, IsRequiredColumn(FillRequiredStructColumn(inputWidth, RequiredColumns))
@@ -2024,7 +2024,7 @@ private:
const std::vector<ui32> RequiredColumns;
const std::vector<ui32> LeftOutputColumns;
const std::vector<ui32> RightOutputColumns;
- const ui64 MemLimit;
+ const ui64 MemLimit;
const std::optional<ui32> SortedTableOrder;
const std::vector<ui32> KeyColumns;
const std::vector<bool> IsRequiredColumn;
@@ -2134,7 +2134,7 @@ IComputationNode* WrapCommonJoinCore(TCallable& callable, const TComputationNode
for (ui32 i = 0; i < keyColumnsNode->GetValuesCount(); ++i) {
keyColumns.push_back(AS_VALUE(TDataLiteral, keyColumnsNode->GetValue(i))->AsValue().Get<ui32>());
}
-
+
const ui64 memLimit = AS_VALUE(TDataLiteral, callable.GetInput(8))->AsValue().Get<ui64>();
std::optional<ui32> sortedTableOrder;
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp
index 194a4cb15e..85088cdfb7 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_join_dict.cpp
@@ -1,11 +1,11 @@
#include "mkql_join_dict.h"
-
+
#include <ydb/library/yql/minikql/computation/mkql_custom_list.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_program_builder.h>
-
+
namespace NKikimr {
namespace NMiniKQL {
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp
index 613958f74b..7035073cff 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_map_join.cpp
@@ -1,11 +1,11 @@
#include "mkql_map_join.h"
-
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_program_builder.h>
#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
-
+
#include <util/string/cast.h>
namespace NKikimr {
@@ -1356,8 +1356,8 @@ private:
}
private:
- NUdf::TUnboxedValue Stream;
- NUdf::TUnboxedValue Dict;
+ NUdf::TUnboxedValue Stream;
+ NUdf::TUnboxedValue Dict;
TComputationContext& Ctx;
const TSelf* const Self;
};
@@ -1412,8 +1412,8 @@ private:
}
private:
- NUdf::TUnboxedValue Stream;
- NUdf::TUnboxedValue Dict;
+ NUdf::TUnboxedValue Stream;
+ NUdf::TUnboxedValue Dict;
TComputationContext& Ctx;
const TSelf* const Self;
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp
index 2e299e487d..250050120b 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_todict.cpp
@@ -270,23 +270,23 @@ public:
Y_UNUSED(isTuple);
Y_UNUSED(encoded);
Set.reserve(itemsCountHint);
- }
-
+ }
+
void Add(NUdf::TUnboxedValue&& key)
{
Set.emplace(key.Get<T>());
- }
-
+ }
+
NUdf::TUnboxedValue Build()
{
return Ctx.HolderFactory.CreateDirectHashedSingleFixedSetHolder<T>(std::move(Set));
- }
-};
-
-template <typename T>
+ }
+};
+
+template <typename T>
class THashedSingleFixedCompactSetAccumulator {
using TSetType = TValuesDictHashSingleFixedCompactSet<T>;
-
+
TComputationContext& Ctx;
TPagedArena Pool;
TSetType Set;
@@ -413,8 +413,8 @@ public:
template <typename T, bool Multi>
class THashedSingleFixedCompactMapAccumulator;
-
-template <typename T>
+
+template <typename T>
class THashedSingleFixedCompactMapAccumulator<T, false> {
using TMapType = TValuesDictHashSingleFixedCompactMap<T>;
@@ -434,20 +434,20 @@ public:
Y_UNUSED(isTuple);
Y_UNUSED(encoded);
Map.SetMaxLoadFactor(COMPACT_HASH_MAX_LOAD_FACTOR);
- }
-
+ }
+
void Add(NUdf::TUnboxedValue&& key, NUdf::TUnboxedValue&& payload)
{
Map.InsertNew(key.Get<T>(), AddSmallValue(Pool, PayloadPacker.Pack(payload)));
- }
-
+ }
+
NUdf::TUnboxedValue Build()
{
return Ctx.HolderFactory.CreateDirectHashedSingleFixedCompactMapHolder(std::move(Map), std::move(Pool), PayloadType, &Ctx);
- }
-};
-
-template <typename T>
+ }
+};
+
+template <typename T>
class THashedSingleFixedCompactMapAccumulator<T, true> {
using TMapType = TValuesDictHashSingleFixedCompactMultiMap<T>;
@@ -467,19 +467,19 @@ public:
Y_UNUSED(isTuple);
Y_UNUSED(encoded);
Map.SetMaxLoadFactor(COMPACT_HASH_MAX_LOAD_FACTOR);
- }
-
+ }
+
void Add(NUdf::TUnboxedValue&& key, NUdf::TUnboxedValue&& payload)
{
Map.Insert(key.Get<T>(), AddSmallValue(Pool, PayloadPacker.Pack(payload)));
- }
-
+ }
+
NUdf::TUnboxedValue Build()
{
return Ctx.HolderFactory.CreateDirectHashedSingleFixedCompactMultiMapHolder(std::move(Map), std::move(Pool), PayloadType, &Ctx);
- }
-};
-
+ }
+};
+
class TSortedSetAccumulator {
TComputationContext& Ctx;
TType* KeyType;
@@ -1776,54 +1776,54 @@ IComputationNode* WrapToHashedDictInternal(TCallable& callable, const TComputati
const auto payloadSelectorNode = callable.GetInput(callable.GetInputsCount() - 4U);
if (!multi && payloadType->IsVoid()) {
- if (isCompact) {
+ if (isCompact) {
if (keyType->IsData()) {
-#define USE_HASHED_SINGLE_FIXED_COMPACT_SET(xType, xLayoutType) \
+#define USE_HASHED_SINGLE_FIXED_COMPACT_SET(xType, xLayoutType) \
case NUdf::TDataType<xType>::Id: \
return WrapToSet< \
THashedSingleFixedCompactSetAccumulator<xLayoutType>>(callable, ctx.NodeLocator, ctx.Mutables);
switch (AS_TYPE(TDataType, keyType)->GetSchemeType()) {
- KNOWN_FIXED_VALUE_TYPES(USE_HASHED_SINGLE_FIXED_COMPACT_SET)
+ KNOWN_FIXED_VALUE_TYPES(USE_HASHED_SINGLE_FIXED_COMPACT_SET)
}
-#undef USE_HASHED_SINGLE_FIXED_COMPACT_SET
+#undef USE_HASHED_SINGLE_FIXED_COMPACT_SET
}
return WrapToSet<THashedCompactSetAccumulator>(callable, ctx.NodeLocator, ctx.Mutables);
- }
-
- if (keyType->IsData()) {
-#define USE_HASHED_SINGLE_FIXED_SET(xType, xLayoutType) \
+ }
+
+ if (keyType->IsData()) {
+#define USE_HASHED_SINGLE_FIXED_SET(xType, xLayoutType) \
case NUdf::TDataType<xType>::Id: \
return WrapToSet< \
THashedSingleFixedSetAccumulator<xLayoutType>>(callable, ctx.NodeLocator, ctx.Mutables);
-
- switch (AS_TYPE(TDataType, keyType)->GetSchemeType()) {
- KNOWN_FIXED_VALUE_TYPES(USE_HASHED_SINGLE_FIXED_SET)
+
+ switch (AS_TYPE(TDataType, keyType)->GetSchemeType()) {
+ KNOWN_FIXED_VALUE_TYPES(USE_HASHED_SINGLE_FIXED_SET)
}
-#undef USE_HASHED_SINGLE_FIXED_SET
+#undef USE_HASHED_SINGLE_FIXED_SET
}
return WrapToSet<THashedSetAccumulator>(callable, ctx.NodeLocator, ctx.Mutables);
}
if (isCompact) {
- if (keyType->IsData()) {
-#define USE_HASHED_SINGLE_FIXED_COMPACT_MAP(xType, xLayoutType) \
+ if (keyType->IsData()) {
+#define USE_HASHED_SINGLE_FIXED_COMPACT_MAP(xType, xLayoutType) \
case NUdf::TDataType<xType>::Id: \
- if (multi) { \
+ if (multi) { \
return WrapToMap< \
THashedSingleFixedCompactMapAccumulator<xLayoutType, true>>(callable, ctx.NodeLocator, ctx.Mutables); \
- } else { \
+ } else { \
return WrapToMap< \
THashedSingleFixedCompactMapAccumulator<xLayoutType, false>>(callable, ctx.NodeLocator, ctx.Mutables); \
- }
-
- switch (AS_TYPE(TDataType, keyType)->GetSchemeType()) {
- KNOWN_FIXED_VALUE_TYPES(USE_HASHED_SINGLE_FIXED_COMPACT_MAP)
- }
-#undef USE_HASHED_SINGLE_FIXED_COMPACT_MAP
- }
-
+ }
+
+ switch (AS_TYPE(TDataType, keyType)->GetSchemeType()) {
+ KNOWN_FIXED_VALUE_TYPES(USE_HASHED_SINGLE_FIXED_COMPACT_MAP)
+ }
+#undef USE_HASHED_SINGLE_FIXED_COMPACT_MAP
+ }
+
if (multi) {
return WrapToMap<THashedCompactMapAccumulator<true>>(callable, ctx.NodeLocator, ctx.Mutables);
} else {
@@ -1832,20 +1832,20 @@ IComputationNode* WrapToHashedDictInternal(TCallable& callable, const TComputati
}
if (keyType->IsData()) {
-#define USE_HASHED_SINGLE_FIXED_MAP(xType, xLayoutType) \
+#define USE_HASHED_SINGLE_FIXED_MAP(xType, xLayoutType) \
case NUdf::TDataType<xType>::Id: \
- if (multi) { \
+ if (multi) { \
return WrapToMap< \
THashedSingleFixedMultiMapAccumulator<xLayoutType>>(callable, ctx.NodeLocator, ctx.Mutables); \
- } else { \
+ } else { \
return WrapToMap< \
THashedSingleFixedMapAccumulator<xLayoutType>>(callable, ctx.NodeLocator, ctx.Mutables); \
- }
+ }
switch (AS_TYPE(TDataType, keyType)->GetSchemeType()) {
- KNOWN_FIXED_VALUE_TYPES(USE_HASHED_SINGLE_FIXED_MAP)
+ KNOWN_FIXED_VALUE_TYPES(USE_HASHED_SINGLE_FIXED_MAP)
}
-#undef USE_HASHED_SINGLE_FIXED_MAP
+#undef USE_HASHED_SINGLE_FIXED_MAP
}
if (multi) {
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp
index 893cb08456..c42ed9c8cf 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_udf.cpp
@@ -14,43 +14,43 @@ namespace NMiniKQL {
namespace {
-class TUdfRunCodegeneratorNode: public TUnboxedImmutableRunCodegeneratorNode {
-public:
- TUdfRunCodegeneratorNode(TMemoryUsageInfo* memInfo,
- NUdf::TUnboxedValue&& value, const TString& moduleIRUniqID, const TString& moduleIR, const TString& functionName)
- : TUnboxedImmutableRunCodegeneratorNode(memInfo, std::move(value))
- , ModuleIRUniqID(moduleIRUniqID)
- , ModuleIR(moduleIR)
- , FunctionName(functionName)
- {
- }
-
-#ifndef MKQL_DISABLE_CODEGEN
+class TUdfRunCodegeneratorNode: public TUnboxedImmutableRunCodegeneratorNode {
+public:
+ TUdfRunCodegeneratorNode(TMemoryUsageInfo* memInfo,
+ NUdf::TUnboxedValue&& value, const TString& moduleIRUniqID, const TString& moduleIR, const TString& functionName)
+ : TUnboxedImmutableRunCodegeneratorNode(memInfo, std::move(value))
+ , ModuleIRUniqID(moduleIRUniqID)
+ , ModuleIR(moduleIR)
+ , FunctionName(functionName)
+ {
+ }
+
+#ifndef MKQL_DISABLE_CODEGEN
void CreateRun(const TCodegenContext& ctx, BasicBlock*& block, Value* result, Value* args) const final {
- ctx.Codegen->LoadBitCode(ModuleIR, ModuleIRUniqID);
-
- auto& context = ctx.Codegen->GetContext();
-
- const auto type = Type::getInt128Ty(context);
- YQL_ENSURE(result->getType() == PointerType::getUnqual(type));
-
- const auto data = ConstantInt::get(Type::getInt64Ty(context), reinterpret_cast<ui64>(UnboxedValue.AsBoxed().Get()));
- const auto ptrStructType = PointerType::getUnqual(StructType::get(context));
+ ctx.Codegen->LoadBitCode(ModuleIR, ModuleIRUniqID);
+
+ auto& context = ctx.Codegen->GetContext();
+
+ const auto type = Type::getInt128Ty(context);
+ YQL_ENSURE(result->getType() == PointerType::getUnqual(type));
+
+ const auto data = ConstantInt::get(Type::getInt64Ty(context), reinterpret_cast<ui64>(UnboxedValue.AsBoxed().Get()));
+ const auto ptrStructType = PointerType::getUnqual(StructType::get(context));
const auto boxed = CastInst::Create(Instruction::IntToPtr, data, ptrStructType, "boxed", block);
- const auto builder = ctx.GetBuilder();
-
- const auto funType = FunctionType::get(Type::getVoidTy(context), {boxed->getType(), result->getType(), builder->getType(), args->getType()}, false);
+ const auto builder = ctx.GetBuilder();
+
+ const auto funType = FunctionType::get(Type::getVoidTy(context), {boxed->getType(), result->getType(), builder->getType(), args->getType()}, false);
const auto runFunc = ctx.Codegen->GetModule().getOrInsertFunction(llvm::StringRef(FunctionName.data(), FunctionName.size()), funType).getCallee();
- CallInst::Create(runFunc, {boxed, result, builder, args}, "", block);
- }
-#endif
-private:
- const TString ModuleIRUniqID;
- const TString ModuleIR;
- const TString FunctionName;
-};
-
-
+ CallInst::Create(runFunc, {boxed, result, builder, args}, "", block);
+ }
+#endif
+private:
+ const TString ModuleIRUniqID;
+ const TString ModuleIR;
+ const TString FunctionName;
+};
+
+
template<class TValidatePolicy, class TValidateMode>
class TUdfWrapper: public TMutableCodegeneratorPtrNode<TUdfWrapper<TValidatePolicy,TValidateMode>> {
typedef TMutableCodegeneratorPtrNode<TUdfWrapper<TValidatePolicy,TValidateMode>> TBaseComputation;
@@ -118,10 +118,10 @@ private:
inline IComputationNode* CreateUdfWrapper(
const TComputationNodeFactoryContext& ctx,
NUdf::TUnboxedValue&& functionImpl,
- TString&& functionName,
- IComputationNode* runConfigNode,
- const TCallableType* callableType)
-{
+ TString&& functionName,
+ IComputationNode* runConfigNode,
+ const TCallableType* callableType)
+{
const auto node = ctx.NodeFactory.CreateImmutableNode(std::move(functionImpl));
ctx.NodePushBack(node);
switch (ctx.ValidateMode) {
@@ -188,20 +188,20 @@ IComputationNode* WrapUdf(TCallable& callable, const TComputationNodeFactoryCont
NUdf::TUnboxedValue impl(NUdf::TUnboxedValuePod(funcInfo.Implementation.Release()));
if (runConfigType->IsVoid()) {
// use function implementation as is
-
- if (ctx.ValidateMode == NUdf::EValidateMode::None && funcInfo.ModuleIR && funcInfo.IRFunctionName) {
- return new TUdfRunCodegeneratorNode(&ctx.HolderFactory.GetMemInfo(), std::move(impl), funcInfo.ModuleIRUniqID, funcInfo.ModuleIR, funcInfo.IRFunctionName);
- }
-
- if (ctx.ValidateMode != NUdf::EValidateMode::None) {
- if (ctx.ValidateMode == NUdf::EValidateMode::Lazy) {
- if (ctx.ValidatePolicy == NUdf::EValidatePolicy::Fail) {
+
+ if (ctx.ValidateMode == NUdf::EValidateMode::None && funcInfo.ModuleIR && funcInfo.IRFunctionName) {
+ return new TUdfRunCodegeneratorNode(&ctx.HolderFactory.GetMemInfo(), std::move(impl), funcInfo.ModuleIRUniqID, funcInfo.ModuleIR, funcInfo.IRFunctionName);
+ }
+
+ if (ctx.ValidateMode != NUdf::EValidateMode::None) {
+ if (ctx.ValidateMode == NUdf::EValidateMode::Lazy) {
+ if (ctx.ValidatePolicy == NUdf::EValidatePolicy::Fail) {
TValidate<TValidateErrorPolicyFail, TValidateModeLazy<TValidateErrorPolicyFail>>::WrapCallable(funcInfo.FunctionType, impl, TStringBuilder() << "FunctionWrapper<" << funcName << ">");
} else {
TValidate<TValidateErrorPolicyThrow, TValidateModeLazy<TValidateErrorPolicyThrow>>::WrapCallable(funcInfo.FunctionType, impl, TStringBuilder() << "FunctionWrapper<" << funcName << ">");
}
} else {
- if (ctx.ValidatePolicy == NUdf::EValidatePolicy::Fail) {
+ if (ctx.ValidatePolicy == NUdf::EValidatePolicy::Fail) {
TValidate<TValidateErrorPolicyFail, TValidateModeGreedy<TValidateErrorPolicyFail>>::WrapCallable(funcInfo.FunctionType, impl, TStringBuilder() << "FunctionWrapper<" << funcName << ">");
} else {
TValidate<TValidateErrorPolicyThrow, TValidateModeGreedy<TValidateErrorPolicyThrow>>::WrapCallable(funcInfo.FunctionType, impl, TStringBuilder() << "FunctionWrapper<" << funcName << ">");
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp
index d4e097e84b..f02c191b20 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_varitem.cpp
@@ -1,35 +1,35 @@
-#include "mkql_varitem.h"
-
+#include "mkql_varitem.h"
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h>
-
+
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_node_builder.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
+
+namespace NKikimr {
+namespace NMiniKQL {
+
namespace {
-template <bool IsOptional>
+template <bool IsOptional>
class TVariantItemWrapper: public TMutableCodegeneratorPtrNode<TVariantItemWrapper<IsOptional>> {
typedef TMutableCodegeneratorPtrNode<TVariantItemWrapper<IsOptional>> TBaseComputation;
-public:
- TVariantItemWrapper(TComputationMutables& mutables, EValueRepresentation kind, IComputationNode* varNode)
- : TBaseComputation(mutables, kind)
- , VarNode(varNode)
- {
- }
-
- NUdf::TUnboxedValuePod DoCalculate(TComputationContext& compCtx) const {
- auto var = VarNode->GetValue(compCtx);
-
- if (IsOptional && !var) {
- return NUdf::TUnboxedValuePod();
- }
-
- return var.Release().GetVariantItem().Release();
- }
-
+public:
+ TVariantItemWrapper(TComputationMutables& mutables, EValueRepresentation kind, IComputationNode* varNode)
+ : TBaseComputation(mutables, kind)
+ , VarNode(varNode)
+ {
+ }
+
+ NUdf::TUnboxedValuePod DoCalculate(TComputationContext& compCtx) const {
+ auto var = VarNode->GetValue(compCtx);
+
+ if (IsOptional && !var) {
+ return NUdf::TUnboxedValuePod();
+ }
+
+ return var.Release().GetVariantItem().Release();
+ }
+
#ifndef MKQL_DISABLE_CODEGEN
void DoGenerateGetValue(const TCodegenContext& ctx, Value* pointer, BasicBlock*& block) const {
auto& context = ctx.Codegen->GetContext();
@@ -76,29 +76,29 @@ public:
block = done;
}
#endif
-private:
- void RegisterDependencies() const final {
- this->DependsOn(VarNode);
- }
-
+private:
+ void RegisterDependencies() const final {
+ this->DependsOn(VarNode);
+ }
+
IComputationNode *const VarNode;
-};
-
+};
+
}
-IComputationNode* WrapVariantItem(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- MKQL_ENSURE(callable.GetInputsCount() == 1, "Expected 1 argument");
- bool isOptional;
+IComputationNode* WrapVariantItem(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
+ MKQL_ENSURE(callable.GetInputsCount() == 1, "Expected 1 argument");
+ bool isOptional;
const auto unpacked = UnpackOptional(callable.GetInput(0), isOptional);
const auto varType = AS_TYPE(TVariantType, unpacked);
-
+
const auto variant = LocateNode(ctx.NodeLocator, callable, 0);
- if (isOptional) {
- return new TVariantItemWrapper<true>(ctx.Mutables, GetValueRepresentation(varType->GetAlternativeType(0)), variant);
- } else {
- return new TVariantItemWrapper<false>(ctx.Mutables, GetValueRepresentation(varType->GetAlternativeType(0)), variant);
- }
-}
-
-}
-}
+ if (isOptional) {
+ return new TVariantItemWrapper<true>(ctx.Mutables, GetValueRepresentation(varType->GetAlternativeType(0)), variant);
+ } else {
+ return new TVariantItemWrapper<false>(ctx.Mutables, GetValueRepresentation(varType->GetAlternativeType(0)), variant);
+ }
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_varitem.h b/ydb/library/yql/minikql/comp_nodes/mkql_varitem.h
index c8be4381f4..f17b735f77 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_varitem.h
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_varitem.h
@@ -1,10 +1,10 @@
-#pragma once
+#pragma once
#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-IComputationNode* WrapVariantItem(TCallable& callable, const TComputationNodeFactoryContext& ctx);
-
-}
-}
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+IComputationNode* WrapVariantItem(TCallable& callable, const TComputationNodeFactoryContext& ctx);
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp b/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp
index 76adbc982b..7b50a385cd 100644
--- a/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/mkql_visitall.cpp
@@ -320,8 +320,8 @@ IComputationNode* WrapVisitAll(TCallable& callable, const TComputationNodeFactor
args.reserve(varType->GetAlternativesCount());
for (auto i = 0U; i < varType->GetAlternativesCount() << 1U; ++i) {
args.emplace_back(LocateExternalNode(ctx.NodeLocator, callable, ++i));
- }
-
+ }
+
if (const auto type = callable.GetType()->GetReturnType(); type->IsFlow()) {
TComputationWideFlowNodePtrVector wideNodes;
wideNodes.reserve(newNodes.size());
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp
index 8d9d557669..3ad551ed20 100644
--- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_chopper_ut.cpp
@@ -1,28 +1,28 @@
#include "mkql_computation_node_ut.h"
-
+
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace {
-
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
template<bool UseLLVM>
TRuntimeNode MakeStream(TSetup<UseLLVM>& setup, ui64 count = 9U) {
TProgramBuilder& pb = *setup.PgmBuilder;
-
- TCallableBuilder callableBuilder(*setup.Env, "TestStream",
+
+ TCallableBuilder callableBuilder(*setup.Env, "TestStream",
pb.NewStreamType(
pb.NewDataType(NUdf::EDataSlot::Uint64)
- )
- );
-
+ )
+ );
+
callableBuilder.Add(pb.NewDataLiteral(count));
-
- return TRuntimeNode(callableBuilder.Build(), false);
-}
-
+
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
template<bool UseLLVM>
TRuntimeNode MakeFlow(TSetup<UseLLVM>& setup, ui64 count = 9U) {
TProgramBuilder& pb = *setup.PgmBuilder;
@@ -45,20 +45,20 @@ TRuntimeNode GroupWithBomb(TSetup<UseLLVM>& setup, TRuntimeNode stream) {
template<bool UseLLVM>
TRuntimeNode Group(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) {
TProgramBuilder& pb = *setup.PgmBuilder;
-
+
const auto keyExtractor = [&](TRuntimeNode item) { return item; };
-
+
return pb.Chopper(stream, keyExtractor, groupSwitch, [&](TRuntimeNode key, TRuntimeNode grpItem) {
return pb.Condense(grpItem, pb.NewDataLiteral<NUdf::EDataSlot::String>("*"),
[&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); },
- [&] (TRuntimeNode item, TRuntimeNode state) {
+ [&] (TRuntimeNode item, TRuntimeNode state) {
auto res = pb.Concat(pb.ToString(key), pb.ToString(item));
res = pb.Concat(state, res);
return pb.Concat(res, pb.NewDataLiteral<NUdf::EDataSlot::String>("*"));
});
- });
-}
-
+ });
+}
+
template<bool UseLLVM>
TRuntimeNode GroupGetKeysFirst(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) {
TProgramBuilder& pb = *setup.PgmBuilder;
@@ -85,20 +85,20 @@ TRuntimeNode GroupGetKeysFirst(TSetup<UseLLVM>& setup, TRuntimeNode stream, cons
template<bool UseLLVM>
TRuntimeNode GroupKeys(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) {
TProgramBuilder& pb = *setup.PgmBuilder;
-
+
const auto keyExtractor = [&](TRuntimeNode item) { return item; };
-
+
return pb.Chopper(stream, keyExtractor, groupSwitch,
[&](TRuntimeNode key, TRuntimeNode group) {
return pb.Map(pb.Take(group, pb.NewDataLiteral<ui64>(1ULL)), [&](TRuntimeNode) { return pb.ToString(key); });
}
);
-}
-
+}
+
template<bool UseLLVM>
TRuntimeNode StreamToString(TSetup<UseLLVM>& setup, TRuntimeNode stream) {
TProgramBuilder& pb = *setup.PgmBuilder;
-
+
stream = pb.Condense(stream, pb.NewDataLiteral<NUdf::EDataSlot::String>("|"),
[&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral(false); },
[&] (TRuntimeNode item, TRuntimeNode state) {
@@ -109,14 +109,14 @@ TRuntimeNode StreamToString(TSetup<UseLLVM>& setup, TRuntimeNode stream) {
stream = pb.FromFlow(stream);
}
return stream;
-}
-
-} // unnamed
-
+}
+
+} // unnamed
+
Y_UNIT_TEST_SUITE(TMiniKQLChopperStreamTest) {
Y_UNIT_TEST_LLVM(TestEmpty) {
TSetup<LLVM> setup;
-
+
const auto stream = GroupWithBomb(setup, MakeStream(setup, 0U));
const auto pgm = StreamToString(setup, stream);
const auto graph = setup.BuildGraph(pgm);
@@ -130,21 +130,21 @@ Y_UNIT_TEST_SUITE(TMiniKQLChopperStreamTest) {
Y_UNIT_TEST_LLVM(TestGrouping) {
TSetup<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
+
+ auto stream = MakeStream(setup);
stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
Y_UNUSED(key);
return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
- });
+ });
const auto pgm = StreamToString(setup, stream);
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
- }
-
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
+ }
+
Y_UNIT_TEST_LLVM(TestGroupingGetKeysFirst) {
TSetup<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
@@ -183,120 +183,120 @@ Y_UNIT_TEST_SUITE(TMiniKQLChopperStreamTest) {
Y_UNIT_TEST_LLVM(TestGroupingWithEmptyInput) {
TSetup<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup, 0);
+
+ auto stream = MakeStream(setup, 0);
stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
Y_UNUSED(key);
return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
- });
+ });
const auto pgm = StreamToString(setup, stream);
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|");
- }
-
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|");
+ }
+
Y_UNIT_TEST_LLVM(TestSingleGroup) {
TSetup<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
+
+ auto stream = MakeStream(setup);
stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
Y_UNUSED(key);
- Y_UNUSED(item);
+ Y_UNUSED(item);
return pb.NewDataLiteral<bool>(false);
- });
+ });
const auto pgm = StreamToString(setup, stream);
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*01*00*00*00*01*02*03*|");
- }
-
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*01*00*00*00*01*02*03*|");
+ }
+
Y_UNIT_TEST_LLVM(TestGroupingWithYield) {
TSetup<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
- TSwitchInput switchInput;
- switchInput.Indicies.push_back(0);
- switchInput.InputType = stream.GetStaticType();
-
+
+ auto stream = MakeStream(setup);
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
stream = pb.Switch(stream,
- MakeArrayRef(&switchInput, 1),
- [&](ui32 /*index*/, TRuntimeNode item1) {
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item1) {
return Group(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) {
Y_UNUSED(key);
return pb.Equals(item2, pb.NewDataLiteral<ui64>(0));
- });
- },
- 1,
+ });
+ },
+ 1,
pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::String))
- );
-
+ );
+
const auto pgm = StreamToString(setup, stream);
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
- }
-
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
+ }
+
Y_UNIT_TEST_LLVM(TestGroupingWithCutSubStreams) {
TSetup<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
-
+
+ auto stream = MakeStream(setup);
+
stream = GroupKeys(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
Y_UNUSED(key);
return pb.Equals(item, pb.NewDataLiteral<ui64>(0));
- });
-
+ });
+
const auto pgm = StreamToString(setup, stream);
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
- }
-
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
+ }
+
Y_UNIT_TEST_LLVM(TestGroupingWithYieldAndCutSubStreams) {
TSetup<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
- TSwitchInput switchInput;
- switchInput.Indicies.push_back(0);
- switchInput.InputType = stream.GetStaticType();
-
+
+ auto stream = MakeStream(setup);
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
stream = pb.Switch(stream,
- MakeArrayRef(&switchInput, 1),
- [&](ui32 /*index*/, TRuntimeNode item1) {
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item1) {
return GroupKeys(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) {
Y_UNUSED(key);
return pb.Equals(item2, pb.NewDataLiteral<ui64>(0));
- });
- },
- 1,
+ });
+ },
+ 1,
pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::String))
- );
-
+ );
+
const auto pgm = StreamToString(setup, stream);
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
- }
-}
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
+ }
+}
#if !defined(MKQL_RUNTIME_VERSION) || MKQL_RUNTIME_VERSION >= 9u
Y_UNIT_TEST_SUITE(TMiniKQLChopperFlowTest) {
Y_UNIT_TEST_LLVM(TestEmpty) {
@@ -315,7 +315,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLChopperFlowTest) {
Y_UNIT_TEST_LLVM(TestGrouping) {
TSetup<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
-
+
auto stream = MakeFlow(setup);
stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
Y_UNUSED(key);
@@ -483,5 +483,5 @@ Y_UNIT_TEST_SUITE(TMiniKQLChopperFlowTest) {
}
}
#endif
-} // NMiniKQL
-} // NKikimr
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp
index 0a147c619d..dc9f1183c4 100644
--- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_combine_ut.cpp
@@ -1,215 +1,215 @@
#include "mkql_computation_node_ut.h"
-
+
#include <ydb/library/yql/minikql/mkql_runtime_version.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
-
-#include <cstring>
+
+#include <cstring>
#include <random>
#include <ctime>
#include <algorithm>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace {
-
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
ui64 g_Yield = std::numeric_limits<ui64>::max();
-ui64 g_TestStreamData[] = {0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2};
+ui64 g_TestStreamData[] = {0, 1, 2, 0, 1, 2, 0, 1, 2, 0, 1, 2};
ui64 g_TestYieldStreamData[] = {0, 1, 2, g_Yield, 0, g_Yield, 1, 2, 0, 1, 2, 0, g_Yield, 1, 2};
-
+
template <bool WithYields>
class TTestStreamWrapper: public TMutableComputationNode<TTestStreamWrapper<WithYields>> {
typedef TMutableComputationNode<TTestStreamWrapper<WithYields>> TBaseComputation;
-public:
- class TStreamValue : public TComputationValue<TStreamValue> {
- public:
- using TBase = TComputationValue<TStreamValue>;
-
+public:
+ class TStreamValue : public TComputationValue<TStreamValue> {
+ public:
+ using TBase = TComputationValue<TStreamValue>;
+
TStreamValue(TMemoryUsageInfo* memInfo, TComputationContext& compCtx, const TTestStreamWrapper* parent)
- : TBase(memInfo)
+ : TBase(memInfo)
, CompCtx(compCtx)
- , Parent(parent)
- {
- }
-
- private:
- NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
+ , Parent(parent)
+ {
+ }
+
+ private:
+ NUdf::EFetchStatus Fetch(NUdf::TUnboxedValue& result) override {
constexpr auto size = WithYields ? Y_ARRAY_SIZE(g_TestYieldStreamData) : Y_ARRAY_SIZE(g_TestStreamData);
if (Index == size) {
- return NUdf::EFetchStatus::Finish;
- }
-
+ return NUdf::EFetchStatus::Finish;
+ }
+
const auto val = WithYields ? g_TestYieldStreamData[Index] : g_TestStreamData[Index];
if (g_Yield == val) {
++Index;
return NUdf::EFetchStatus::Yield;
}
- NUdf::TUnboxedValue* items = nullptr;
+ NUdf::TUnboxedValue* items = nullptr;
result = CompCtx.HolderFactory.CreateDirectArrayHolder(2, items);
items[0] = NUdf::TUnboxedValuePod(val);
- if (((Index + 1) % Parent->PeakStep) == 0) {
+ if (((Index + 1) % Parent->PeakStep) == 0) {
auto str = MakeStringNotFilled(64ul << 20);
- const auto& buf = str.AsStringRef();
- memset(buf.Data(), ' ', buf.Size());
+ const auto& buf = str.AsStringRef();
+ memset(buf.Data(), ' ', buf.Size());
items[1] = std::move(str);
- } else {
+ } else {
items[1] = NUdf::TUnboxedValuePod::Zero();
- }
-
- ++Index;
- return NUdf::EFetchStatus::Ok;
- }
-
- private:
+ }
+
+ ++Index;
+ return NUdf::EFetchStatus::Ok;
+ }
+
+ private:
TComputationContext& CompCtx;
const TTestStreamWrapper* const Parent;
- ui64 Index = 0;
- };
-
+ ui64 Index = 0;
+ };
+
TTestStreamWrapper(TComputationMutables& mutables, ui64 peakStep)
: TBaseComputation(mutables)
, PeakStep(peakStep)
- {
- }
-
+ {
+ }
+
NUdf::TUnboxedValuePod DoCalculate(TComputationContext& ctx) const {
return ctx.HolderFactory.Create<TStreamValue>(ctx, this);
- }
-
-private:
+ }
+
+private:
void RegisterDependencies() const final {
- }
-
-private:
- const ui64 PeakStep;
-};
-
+ }
+
+private:
+ const ui64 PeakStep;
+};
+
template <bool WithYields>
IComputationNode* WrapTestStream(TCallable& callable, const TComputationNodeFactoryContext& ctx) {
- MKQL_ENSURE(callable.GetInputsCount() == 1, "Expected 1 args");
- const ui64 peakStep = AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().Get<ui64>();
+ MKQL_ENSURE(callable.GetInputsCount() == 1, "Expected 1 args");
+ const ui64 peakStep = AS_VALUE(TDataLiteral, callable.GetInput(0))->AsValue().Get<ui64>();
return new TTestStreamWrapper<WithYields>(ctx.Mutables, peakStep);
-}
-
-TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
- return CreateDeterministicRandomProvider(1);
-}
-
-TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
+}
+
+TIntrusivePtr<IRandomProvider> CreateRandomProvider() {
+ return CreateDeterministicRandomProvider(1);
+}
+
+TIntrusivePtr<ITimeProvider> CreateTimeProvider() {
return CreateDeterministicTimeProvider(10000000);
-}
-
-TComputationNodeFactory GetTestFactory() {
- return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
+}
+
+TComputationNodeFactory GetTestFactory() {
+ return [](TCallable& callable, const TComputationNodeFactoryContext& ctx) -> IComputationNode* {
if (callable.GetType()->GetName() == "TestList") {
return new TExternalComputationNode(ctx.Mutables);
}
- if (callable.GetType()->GetName() == "TestStream") {
+ if (callable.GetType()->GetName() == "TestStream") {
return WrapTestStream<false>(callable, ctx);
- }
+ }
if (callable.GetType()->GetName() == "TestYieldStream") {
return WrapTestStream<true>(callable, ctx);
}
- return GetBuiltinFactory()(callable, ctx);
- };
-}
-
+ return GetBuiltinFactory()(callable, ctx);
+ };
+}
+
template<bool UseLLVM>
struct TSetup_ {
TSetup_() {
- FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
- RandomProvider = CreateRandomProvider();
- TimeProvider = CreateTimeProvider();
-
- Env.Reset(new TTypeEnvironment(Alloc));
- PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
- }
-
+ FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ RandomProvider = CreateRandomProvider();
+ TimeProvider = CreateTimeProvider();
+
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
TAutoPtr<IComputationGraph> BuildGraph(TRuntimeNode pgm, EGraphPerProcess graphPerProcess = EGraphPerProcess::Multi, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
- Explorer.Walk(pgm.GetNode(), *Env);
+ Explorer.Walk(pgm.GetNode(), *Env);
TComputationPatternOpts opts(Alloc.Ref(), *Env, GetTestFactory(), FunctionRegistry.Get(),
NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, UseLLVM ? "" : "OFF", graphPerProcess);
Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
return Pattern->Clone(opts.ToComputationOptions(*RandomProvider, *TimeProvider));
- }
-
- TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
- TIntrusivePtr<IRandomProvider> RandomProvider;
- TIntrusivePtr<ITimeProvider> TimeProvider;
-
- TScopedAlloc Alloc;
- THolder<TTypeEnvironment> Env;
- THolder<TProgramBuilder> PgmBuilder;
-
- TExploringNodeVisitor Explorer;
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+
+ TScopedAlloc Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
IComputationPattern::TPtr Pattern;
-};
-
+};
+
template <bool LLVM, bool WithYields = false>
TRuntimeNode MakeStream(TSetup_<LLVM>& setup, ui64 peakStep) {
TProgramBuilder& pb = *setup.PgmBuilder;
-
+
TCallableBuilder callableBuilder(*setup.Env, WithYields ? "TestYieldStream" : "TestStream",
pb.NewStreamType(
pb.NewStructType({
{TStringBuf("a"), pb.NewDataType(NUdf::EDataSlot::Uint64)},
{TStringBuf("b"), pb.NewDataType(NUdf::EDataSlot::String)}
- })
- )
- );
+ })
+ )
+ );
callableBuilder.Add(pb.NewDataLiteral(peakStep));
-
- return TRuntimeNode(callableBuilder.Build(), false);
-}
-
+
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
template <bool OverFlow>
TRuntimeNode Combine(TProgramBuilder& pb, TRuntimeNode stream, std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)> finishLambda) {
const auto keyExtractor = [&](TRuntimeNode item) {
return pb.Member(item, "a");
- };
+ };
const auto init = [&](TRuntimeNode /*key*/, TRuntimeNode item) {
- return item;
- };
+ return item;
+ };
const auto update = [&](TRuntimeNode /*key*/, TRuntimeNode item, TRuntimeNode state) {
const auto a = pb.Add(pb.Member(item, "a"), pb.Member(state, "a"));
const auto b = pb.Concat(pb.Member(item, "b"), pb.Member(state, "b"));
return pb.NewStruct({
{TStringBuf("a"), a},
{TStringBuf("b"), b},
- });
- };
-
+ });
+ };
+
return OverFlow ?
pb.FromFlow(pb.CombineCore(pb.ToFlow(stream), keyExtractor, init, update, finishLambda, 64ul << 20)):
pb.CombineCore(stream, keyExtractor, init, update, finishLambda, 64ul << 20);
-}
-
+}
+
TRuntimeNode Reduce(TProgramBuilder& pb, TRuntimeNode stream) {
return pb.Condense(stream, pb.NewDataLiteral<ui64>(0),
[&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); },
[&] (TRuntimeNode item, TRuntimeNode state) { return pb.Add(state, item); }
);
-}
-
+}
+
TRuntimeNode StreamToString(TProgramBuilder& pb, TRuntimeNode stream) {
const auto sorted = pb.Sort(stream, pb.NewDataLiteral(true),
[&](TRuntimeNode item) {
return item;
});
-
+
return pb.Condense(sorted, pb.NewDataLiteral<NUdf::EDataSlot::String>("|"),
[&] (TRuntimeNode, TRuntimeNode) { return pb.NewDataLiteral<bool>(false); },
[&] (TRuntimeNode item, TRuntimeNode state) {
return pb.Concat(pb.Concat(state, pb.ToString(item)), pb.NewDataLiteral<NUdf::EDataSlot::String>("|"));
}
);
-}
-
-} // unnamed
-
+}
+
+} // unnamed
+
Y_UNIT_TEST_SUITE(TMiniKQLCombineStreamTest) {
Y_UNIT_TEST_LLVM(TestFullCombineWithOptOut) {
TSetup_<LLVM> setup;
@@ -217,7 +217,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLCombineStreamTest) {
const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
return pb.NewOptional(pb.Member(state, "a"));
};
-
+
const auto stream = MakeStream(setup, Max<ui64>());
const auto pgm = StreamToString(pb, Combine<false>(pb, stream, finish));
const auto graph = setup.BuildGraph(pgm);
@@ -845,18 +845,18 @@ Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowTest) {
TProgramBuilder& pb = *setup.PgmBuilder;
const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
return pb.NewOptional(pb.Member(state, "a"));
- };
-
+ };
+
const auto stream = MakeStream(setup, Max<ui64>());
const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish));
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
NUdf::TUnboxedValue result;
UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|4|8|");
- }
-
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|4|8|");
+ }
+
Y_UNIT_TEST_LLVM(TestFullCombineWithListOut) {
TSetup_<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
@@ -866,19 +866,19 @@ Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowTest) {
auto list = pb.NewEmptyList(itemType);
list = pb.Append(list, item);
list = pb.Append(list, item);
- return list;
- };
-
+ return list;
+ };
+
const auto stream = MakeStream(setup, Max<ui64>());
const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish));
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
NUdf::TUnboxedValue result;
UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|");
- }
-
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|");
+ }
+
Y_UNIT_TEST_LLVM(TestFullCombineWithStreamOut) {
TSetup_<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
@@ -889,18 +889,18 @@ Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowTest) {
list = pb.Append(list, item);
list = pb.Append(list, item);
return pb.Iterator(list, MakeArrayRef(&state, 1));
- };
-
+ };
+
const auto stream = MakeStream(setup, Max<ui64>());
const auto pgm = StreamToString(pb, Combine<true>(pb, stream, finish));
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
NUdf::TUnboxedValue result;
UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|");
- }
-
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|4|4|8|8|");
+ }
+
Y_UNIT_TEST_LLVM(TestFullCombineWithOptOutAndYields) {
TSetup_<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
@@ -976,74 +976,74 @@ Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowTest) {
TProgramBuilder& pb = *setup.PgmBuilder;
const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
return pb.NewOptional(pb.Member(state, "a"));
- };
-
+ };
+
const auto stream = MakeStream(setup, 6ul);
const auto combine = Combine<true>(pb, stream, finish);
- {
+ {
const auto pgm = Reduce(pb, combine);
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
NUdf::TUnboxedValue result;
UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul);
- }
- {
+
+ UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul);
+ }
+ {
const auto pgm = StreamToString(pb, combine);
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
NUdf::TUnboxedValue result;
UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
+
UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|2|2|4|4|");
- }
- }
-
+ }
+ }
+
Y_UNIT_TEST_LLVM(TestCombineInSingleProc) {
TSetup_<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
return pb.NewOptional(pb.Member(state, "a"));
- };
-
+ };
+
const auto stream = MakeStream(setup, 6ul);
const auto pgm = Reduce(pb, Combine<true>(pb, stream, finish));
const auto graph = setup.BuildGraph(pgm, EGraphPerProcess::Single);
const auto streamVal = graph->GetValue();
NUdf::TUnboxedValue result;
UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul);
- }
-
+
+ UNIT_ASSERT_VALUES_EQUAL(result.Get<ui64>(), 12ul);
+ }
+
Y_UNIT_TEST_LLVM(TestCombineSwithYield) {
TSetup_<LLVM> setup;
TProgramBuilder& pb = *setup.PgmBuilder;
const auto finish = [&](TRuntimeNode /*key*/, TRuntimeNode state) {
return pb.NewOptional(pb.Member(state, "a"));
- };
-
- auto stream = MakeStream(setup, Max<ui64>());
- TSwitchInput switchInput;
- switchInput.Indicies.push_back(0);
- switchInput.InputType = stream.GetStaticType();
-
+ };
+
+ auto stream = MakeStream(setup, Max<ui64>());
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
stream = pb.Switch(stream,
- MakeArrayRef(&switchInput, 1),
+ MakeArrayRef(&switchInput, 1),
[&](ui32 /*index*/, TRuntimeNode item) { return Combine<true>(pb, item, finish); },
- 1,
+ 1,
pb.NewStreamType(pb.NewDataType(NUdf::EDataSlot::Uint64))
- );
-
+ );
+
const auto pgm = StreamToString(pb, stream);
const auto graph = setup.BuildGraph(pgm);
const auto streamVal = graph->GetValue();
NUdf::TUnboxedValue result;
UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
+
UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|1|1|1|1|2|2|2|2|");
- }
+ }
}
Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowPerfTest) {
@@ -1568,7 +1568,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLCombineFlowPerfTest) {
Cerr << "Runtime is " << t2 - t1 << " vs C++ " << cppTime << Endl;
}
-}
+}
#endif
-} // NMiniKQL
-} // NKikimr
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp
index ac6d4c356b..44fd1fddbe 100644
--- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_dict_ut.cpp
@@ -137,7 +137,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLDictRelatedNodesTest) {
},
[&](TRuntimeNode item) {
return pgmBuilder.Member(item, "Payload");
- }, false, 0);
+ }, false, 0);
return dict;
}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp
index 1833bbf509..8c102c4533 100644
--- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_group_ut.cpp
@@ -1,97 +1,97 @@
#include "mkql_computation_node_ut.h"
-
+
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace {
-
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+
template<bool UseLLVM>
TRuntimeNode MakeStream(TSetup<UseLLVM>& setup, ui64 count = 9U) {
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- TCallableBuilder callableBuilder(*setup.Env, "TestStream",
- pgmBuilder.NewStreamType(
- pgmBuilder.NewDataType(NUdf::EDataSlot::Uint64)
- )
- );
-
- callableBuilder.Add(pgmBuilder.NewDataLiteral(count));
-
- return TRuntimeNode(callableBuilder.Build(), false);
-}
-
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ TCallableBuilder callableBuilder(*setup.Env, "TestStream",
+ pgmBuilder.NewStreamType(
+ pgmBuilder.NewDataType(NUdf::EDataSlot::Uint64)
+ )
+ );
+
+ callableBuilder.Add(pgmBuilder.NewDataLiteral(count));
+
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
template<bool UseLLVM>
-TRuntimeNode Group(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch,
- const std::function<TRuntimeNode(TRuntimeNode)>& handler = {})
-{
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto keyExtractor = [&](TRuntimeNode item) {
- return item;
- };
-
- stream = pgmBuilder.GroupingCore(stream, groupSwitch, keyExtractor, handler);
- return pgmBuilder.FlatMap(stream, [&](TRuntimeNode grpItem) {
- return pgmBuilder.Squeeze(pgmBuilder.Nth(grpItem, 1),
- pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("*"),
- [&] (TRuntimeNode item, TRuntimeNode state) {
- auto res = pgmBuilder.Concat(pgmBuilder.ToString(pgmBuilder.Nth(grpItem, 0)), pgmBuilder.ToString(item));
- res = pgmBuilder.Concat(state, res);
- res = pgmBuilder.Concat(res, pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("*"));
- return res;
- },
- {}, {});
- });
-}
-
+TRuntimeNode Group(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch,
+ const std::function<TRuntimeNode(TRuntimeNode)>& handler = {})
+{
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto keyExtractor = [&](TRuntimeNode item) {
+ return item;
+ };
+
+ stream = pgmBuilder.GroupingCore(stream, groupSwitch, keyExtractor, handler);
+ return pgmBuilder.FlatMap(stream, [&](TRuntimeNode grpItem) {
+ return pgmBuilder.Squeeze(pgmBuilder.Nth(grpItem, 1),
+ pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("*"),
+ [&] (TRuntimeNode item, TRuntimeNode state) {
+ auto res = pgmBuilder.Concat(pgmBuilder.ToString(pgmBuilder.Nth(grpItem, 0)), pgmBuilder.ToString(item));
+ res = pgmBuilder.Concat(state, res);
+ res = pgmBuilder.Concat(res, pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("*"));
+ return res;
+ },
+ {}, {});
+ });
+}
+
template<bool UseLLVM>
TRuntimeNode GroupKeys(TSetup<UseLLVM>& setup, TRuntimeNode stream, const std::function<TRuntimeNode(TRuntimeNode, TRuntimeNode)>& groupSwitch) {
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto keyExtractor = [&](TRuntimeNode item) {
- return item;
- };
-
- stream = pgmBuilder.GroupingCore(stream, groupSwitch, keyExtractor);
- return pgmBuilder.Map(stream, [&](TRuntimeNode grpItem) {
- return pgmBuilder.ToString(pgmBuilder.Nth(grpItem, 0));
- });
-}
-
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto keyExtractor = [&](TRuntimeNode item) {
+ return item;
+ };
+
+ stream = pgmBuilder.GroupingCore(stream, groupSwitch, keyExtractor);
+ return pgmBuilder.Map(stream, [&](TRuntimeNode grpItem) {
+ return pgmBuilder.ToString(pgmBuilder.Nth(grpItem, 0));
+ });
+}
+
template<bool UseLLVM>
TRuntimeNode StreamToString(TSetup<UseLLVM>& setup, TRuntimeNode stream) {
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- return pgmBuilder.Squeeze(stream, pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|"), [&] (TRuntimeNode item, TRuntimeNode state) {
- return pgmBuilder.Concat(pgmBuilder.Concat(state, item), pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|"));
- }, {}, {});
-}
-
-} // unnamed
-
-
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ return pgmBuilder.Squeeze(stream, pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|"), [&] (TRuntimeNode item, TRuntimeNode state) {
+ return pgmBuilder.Concat(pgmBuilder.Concat(state, item), pgmBuilder.NewDataLiteral<NUdf::EDataSlot::String>("|"));
+ }, {}, {});
+}
+
+} // unnamed
+
+
Y_UNIT_TEST_SUITE(TMiniKQLGroupingTest) {
Y_UNIT_TEST_LLVM(TestGrouping) {
TSetup<LLVM> setup;
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
Y_UNUSED(key);
return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0));
- });
- auto pgm = StreamToString(setup, stream);
- auto graph = setup.BuildGraph(pgm);
- auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
- }
-
+ });
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
+ }
+
Y_UNIT_TEST_LLVM(TestGroupingKeyNotEquals) {
TSetup<LLVM> setup;
TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
@@ -111,144 +111,144 @@ Y_UNIT_TEST_SUITE(TMiniKQLGroupingTest) {
Y_UNIT_TEST_LLVM(TestGroupingWithEmptyInput) {
TSetup<LLVM> setup;
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup, 0);
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup, 0);
stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
Y_UNUSED(key);
return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0));
- });
- auto pgm = StreamToString(setup, stream);
- auto graph = setup.BuildGraph(pgm);
- auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|");
- }
-
+ });
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|");
+ }
+
Y_UNIT_TEST_LLVM(TestSingleGroup) {
TSetup<LLVM> setup;
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
stream = Group(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
Y_UNUSED(key);
- Y_UNUSED(item);
- return pgmBuilder.NewDataLiteral<bool>(false);
- });
- auto pgm = StreamToString(setup, stream);
- auto graph = setup.BuildGraph(pgm);
- auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*01*00*00*00*01*02*03*|");
- }
-
+ Y_UNUSED(item);
+ return pgmBuilder.NewDataLiteral<bool>(false);
+ });
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*00*01*00*00*00*01*02*03*|");
+ }
+
Y_UNIT_TEST_LLVM(TestGroupingWithYield) {
TSetup<LLVM> setup;
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
- TSwitchInput switchInput;
- switchInput.Indicies.push_back(0);
- switchInput.InputType = stream.GetStaticType();
-
- stream = pgmBuilder.Switch(stream,
- MakeArrayRef(&switchInput, 1),
- [&](ui32 /*index*/, TRuntimeNode item1) {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
+ stream = pgmBuilder.Switch(stream,
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item1) {
return Group(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) {
Y_UNUSED(key);
return pgmBuilder.Equals(item2, pgmBuilder.NewDataLiteral<ui64>(0));
- });
- },
- 1,
- pgmBuilder.NewStreamType(pgmBuilder.NewDataType(NUdf::EDataSlot::String))
- );
-
- auto pgm = StreamToString(setup, stream);
- auto graph = setup.BuildGraph(pgm);
- auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
- }
-
+ });
+ },
+ 1,
+ pgmBuilder.NewStreamType(pgmBuilder.NewDataType(NUdf::EDataSlot::String))
+ );
+
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*00*|*00*01*|*00*|*00*|*00*01*02*03*|");
+ }
+
Y_UNIT_TEST_LLVM(TestGroupingWithoutFetchingSubStreams) {
TSetup<LLVM> setup;
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
-
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+
stream = GroupKeys(setup, stream, [&](TRuntimeNode key, TRuntimeNode item) {
Y_UNUSED(key);
return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0));
- });
-
- auto pgm = StreamToString(setup, stream);
- auto graph = setup.BuildGraph(pgm);
- auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
- }
-
+ });
+
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
+ }
+
Y_UNIT_TEST_LLVM(TestGroupingWithYieldAndWithoutFetchingSubStreams) {
TSetup<LLVM> setup;
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
- TSwitchInput switchInput;
- switchInput.Indicies.push_back(0);
- switchInput.InputType = stream.GetStaticType();
-
- stream = pgmBuilder.Switch(stream,
- MakeArrayRef(&switchInput, 1),
- [&](ui32 /*index*/, TRuntimeNode item1) {
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ TSwitchInput switchInput;
+ switchInput.Indicies.push_back(0);
+ switchInput.InputType = stream.GetStaticType();
+
+ stream = pgmBuilder.Switch(stream,
+ MakeArrayRef(&switchInput, 1),
+ [&](ui32 /*index*/, TRuntimeNode item1) {
return GroupKeys(setup, item1, [&](TRuntimeNode key, TRuntimeNode item2) {
Y_UNUSED(key);
return pgmBuilder.Equals(item2, pgmBuilder.NewDataLiteral<ui64>(0));
- });
- },
- 1,
- pgmBuilder.NewStreamType(pgmBuilder.NewDataType(NUdf::EDataSlot::String))
- );
-
- auto pgm = StreamToString(setup, stream);
- auto graph = setup.BuildGraph(pgm);
- auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
- }
-
- Y_UNIT_TEST_LLVM(TestGroupingWithHandler) {
- TSetup<LLVM> setup;
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
-
- auto stream = MakeStream(setup);
- stream = Group(setup, stream,
- [&](TRuntimeNode key, TRuntimeNode item) {
- Y_UNUSED(key);
- return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0));
- },
- [&](TRuntimeNode item) {
- return pgmBuilder.Add(pgmBuilder.Convert(item, pgmBuilder.NewDataType(NUdf::EDataSlot::Int32)), pgmBuilder.NewDataLiteral<ui64>(1));
- }
- );
- auto pgm = StreamToString(setup, stream);
- auto graph = setup.BuildGraph(pgm);
- auto streamVal = graph->GetValue();
- NUdf::TUnboxedValue result;
- UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
-
- UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*01*|*01*02*|*01*|*01*|*01*02*03*04*|");
- }
-}
-
-} // NMiniKQL
-} // NKikimr
+ });
+ },
+ 1,
+ pgmBuilder.NewStreamType(pgmBuilder.NewDataType(NUdf::EDataSlot::String))
+ );
+
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|0|0|0|0|0|");
+ }
+
+ Y_UNIT_TEST_LLVM(TestGroupingWithHandler) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+
+ auto stream = MakeStream(setup);
+ stream = Group(setup, stream,
+ [&](TRuntimeNode key, TRuntimeNode item) {
+ Y_UNUSED(key);
+ return pgmBuilder.Equals(item, pgmBuilder.NewDataLiteral<ui64>(0));
+ },
+ [&](TRuntimeNode item) {
+ return pgmBuilder.Add(pgmBuilder.Convert(item, pgmBuilder.NewDataType(NUdf::EDataSlot::Int32)), pgmBuilder.NewDataLiteral<ui64>(1));
+ }
+ );
+ auto pgm = StreamToString(setup, stream);
+ auto graph = setup.BuildGraph(pgm);
+ auto streamVal = graph->GetValue();
+ NUdf::TUnboxedValue result;
+ UNIT_ASSERT_EQUAL(streamVal.Fetch(result), NUdf::EFetchStatus::Ok);
+
+ UNIT_ASSERT_VALUES_EQUAL(TStringBuf(result.AsStringRef()), "|*01*|*01*02*|*01*|*01*|*01*02*03*04*|");
+ }
+}
+
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp b/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
index 5a5cd7e2de..e49a2e9d4e 100644
--- a/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
+++ b/ydb/library/yql/minikql/comp_nodes/ut/mkql_todict_ut.cpp
@@ -1,109 +1,109 @@
-#include "mkql_computation_node_ut.h"
-
+#include "mkql_computation_node_ut.h"
+
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
-
-#include <util/random/shuffle.h>
-
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-static const TStringBuf data[] = {
- "13d49d4db08e57d645fe4d44bbed4738f386af6e9e742cf186961063feb9919b",
- "14d285e88582d87c41d3e6d2e9352686d0363ea74a297fe02f901f18c19978a3",
- "1795ad46329c4fc6b3355dc22d252c5fe390a971ddf009b54fdeceb93d3b8930",
- "18042e88fb4cf6b09cb8e6c5588ae525fc7a37bd2248a857d83ac1d1dcdf0a64",
- "1b30b154ac814f7e4ed7e7488e037d781b78fbc336cac027f4c301ad9368514e",
- "1a0b94cebdc038bb293575af4f6954e9dbf19801e581ad03be90b4aef36347d7",
- "1c9ac5b87de7d68efae1bdf1ad47e58d28a6a70e966f19899798c61b2a65b6e2",
- "1618c1e3d9dbc3edaccb7934eca55d2d96cb59d2633655f57401dba99deec3ef",
- "1bd7a6ff86a1940283e202b142ecba685fea86f93f2aafad8cd37d80582aca95",
- "0fba3f2f741b0579a3eec906f5341e0556fbd74088fcdfbe776bd6122fa81681",
- "19768b3228cef7a82e0f1c367d78c42596fa511c735bd85d7cafca0965045562",
- "1a9c0a14272795d7ad39a4725754c3f1d013a761c41fba16e06ae247833fd42b",
- "1562ce72ff7229866723e81e35db30d08c8b8dc7b7d076cff787f157d70763e6",
- "0faf214bafe219413618fdf186bb9290e6a610755d90947cd686b72899e78445",
- "14f3fe97da837197b98f6303ac5aa1b6f34bffe9841fe65f084a27f4bd4ced8a",
- "198c0706af7107ababebf1500875ba64508519b21aa534d0f55e8a32e394799d",
- "1bb66a4593b77b1650a4a530bae30e454c2815477769d67fe6c6b337ae4acafe",
- "0f67ef1ca6ef6b2d699dfac0360e8f24dc8960428cff058fe285d63ab55ef6d3",
- "1097009fe853793802120394fbb6404df82725240d410c69375e5a53ec5395b8",
- "1b1262275eae8a055732253e8d717c19ebde386b25e51dd546413e7ee997c5e1",
- "1c4a73588541a0c869b4ee27c32cc4218f3c8db13713c61cedc387336a2720c9",
- "1c73442f0ac53d8b38f779231680fab806a6cb9c86d25d9db5fa67c0ebf8e803",
- "19152f0c06baf7962ca287a303b85437f321d725985f1586ac8358bdb6a0df63",
- "13436f337815f5929559e6f621b850ed60b36f23ce9d8d06db981b70d40ad3db",
- "298268d866eea5d6fcae470fdbb6d7787d73ab1e50b8126d6452d81264fbdafd",
- "1a67b4e4c213baa140c5a00352cdbc9256b4e2fe81482c380b08ebe2e6b76e1b",
- "19824d2008be54e35a0e2a9d2df9746e96f73367518b111695e1c3857966c241",
- "2997c49ed21482d30b8ef89bd26bfdb6384dda6825032145fe0a3ad9d2f2a7e3",
- "137ccc1d4ab00210bd9af5ee875cb799bd818f4803470abca68a9655ea73be01",
- "12d4cf2eb41c90ede84bece72f76e97d7d0144c45341a0176f656b295cb838c3",
- "11d02da4f449e6aeee4f213409baed6eaab35496688d743991870ba093676c44",
- "163fb1ef04a1453a44fb897014287f7ceefe0b553d99718f986ada32cec6ca29",
- "16f579a7eda4d7f5cde29724bf35e1b36e95fbeb72914ba2ba8f19192b92dab7",
- "0f60c1387bf29d8d81174bd43c136e75f0f33b8b4d2712de0cc3a23f04fac76e",
- "0f83662d3b4cc9aaa0f76c8801d2d32909c050286d514acc108b6c3d9964679b",
- "1a30b7c4bf1c4eaaa92802cb90a27b5073d4a5ec095271490df8443b8f5df88f",
- "105af591b517f126c985f54e833d61907ff49945ab123a606caa6d9bda0e3d66",
- "1a5196fdfc1b81974905a66e6f1ff31403fc84b4d84effde521e848153f42e10",
- "17d6cb5ba9489d8397cb1e1d054e45cff6c7225aeeba5c9e76cacd9da6c9a0c1",
- "127ab4e2169329082bdd546e94c4fb6643999b14a26e08eaa719689789767014",
- "143883410f000b5f3ff4c6434b0654516e9502d0a50a2b3ecdc20c8d3e198915",
- "16ccd345646dd3d39e6bd157b51513c1b319bd1f441199003052a74b2eddb53d",
- "11e9f02dc56d575fac5a306a6e83f597ffda1bd81a01f13fdda059ab24d90892",
- "13f75a9e662faea5fc93f0f83d363c42083345cdcc42f1b0d320d11409ef3052",
- "18cca97e8c6ede52e0b7d8c53c85c0fac68f6d1b7c6622a4cebc21433e6d8eea",
- "160d6b818fab5ad00a1e81e46888c5ff3e5f2c175c013ce17d88c31df4475aba",
- "1c4d09dff19175af7fc0d8e8fd23e9288fc2839dedfc067dcf9f5a3e3a9d92aa",
- "16e25b2a6eef4cde6879c20c94c4360604b9099c29e1abaf9fc079fe67cfcaac",
- "2a577ab7e2541e2cc2cc20e6a76c4ea9b77501808db9c4045be82c680cf227d5",
- "11b4753fd9cc33656dbd59769b3202b7f68bd067bf7f64bd54676f6f60366ef1",
- "1932a0aecc4a569d7d3fbcdd329b92c0b4dbd870d6be48ec4f18285ab3183676",
- "2a2e6b62a4383cb48ffbb69b2f356ceb0410593f5b5500142498692dec7c125f",
-};
-
-
-Y_UNIT_TEST_SUITE(TMiniKQLToDictTest) {
- Y_UNIT_TEST_LLVM(TestCompactUtf8Set) {
- TSetup<LLVM> setup;
- TProgramBuilder& pb = *setup.PgmBuilder;
-
- TVector<TRuntimeNode> items;
- for (auto s: data) {
- items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s));
- }
- Shuffle(items.begin(), items.end());
- auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
- auto list = pb.NewList(dataType, items);
- auto dict = pb.ToHashedDict(list, false, [](TRuntimeNode n) { return n; }, [&pb](TRuntimeNode /*n*/) { return pb.NewVoid(); }, true);
- auto pgmReturn = pb.Contains(dict, items.front());
-
- auto graph = setup.BuildGraph(pgmReturn);
- auto res = graph->GetValue().template Get<bool>();
- UNIT_ASSERT_VALUES_EQUAL(res, true);
- }
-
- Y_UNIT_TEST_LLVM(TestUtf8Set) {
- TSetup<LLVM> setup;
- TProgramBuilder& pb = *setup.PgmBuilder;
-
- TVector<TRuntimeNode> items;
- for (auto s: data) {
- items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s));
- }
- Shuffle(items.begin(), items.end());
- auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
- auto list = pb.NewList(dataType, items);
- auto dict = pb.ToHashedDict(list, false, [](TRuntimeNode n) { return n; }, [&pb](TRuntimeNode /*n*/) { return pb.NewVoid(); }, false);
- auto pgmReturn = pb.Contains(dict, items.front());
-
- auto graph = setup.BuildGraph(pgmReturn);
- auto res = graph->GetValue().template Get<bool>();
- UNIT_ASSERT_VALUES_EQUAL(res, true);
- }
+
+#include <util/random/shuffle.h>
+
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+static const TStringBuf data[] = {
+ "13d49d4db08e57d645fe4d44bbed4738f386af6e9e742cf186961063feb9919b",
+ "14d285e88582d87c41d3e6d2e9352686d0363ea74a297fe02f901f18c19978a3",
+ "1795ad46329c4fc6b3355dc22d252c5fe390a971ddf009b54fdeceb93d3b8930",
+ "18042e88fb4cf6b09cb8e6c5588ae525fc7a37bd2248a857d83ac1d1dcdf0a64",
+ "1b30b154ac814f7e4ed7e7488e037d781b78fbc336cac027f4c301ad9368514e",
+ "1a0b94cebdc038bb293575af4f6954e9dbf19801e581ad03be90b4aef36347d7",
+ "1c9ac5b87de7d68efae1bdf1ad47e58d28a6a70e966f19899798c61b2a65b6e2",
+ "1618c1e3d9dbc3edaccb7934eca55d2d96cb59d2633655f57401dba99deec3ef",
+ "1bd7a6ff86a1940283e202b142ecba685fea86f93f2aafad8cd37d80582aca95",
+ "0fba3f2f741b0579a3eec906f5341e0556fbd74088fcdfbe776bd6122fa81681",
+ "19768b3228cef7a82e0f1c367d78c42596fa511c735bd85d7cafca0965045562",
+ "1a9c0a14272795d7ad39a4725754c3f1d013a761c41fba16e06ae247833fd42b",
+ "1562ce72ff7229866723e81e35db30d08c8b8dc7b7d076cff787f157d70763e6",
+ "0faf214bafe219413618fdf186bb9290e6a610755d90947cd686b72899e78445",
+ "14f3fe97da837197b98f6303ac5aa1b6f34bffe9841fe65f084a27f4bd4ced8a",
+ "198c0706af7107ababebf1500875ba64508519b21aa534d0f55e8a32e394799d",
+ "1bb66a4593b77b1650a4a530bae30e454c2815477769d67fe6c6b337ae4acafe",
+ "0f67ef1ca6ef6b2d699dfac0360e8f24dc8960428cff058fe285d63ab55ef6d3",
+ "1097009fe853793802120394fbb6404df82725240d410c69375e5a53ec5395b8",
+ "1b1262275eae8a055732253e8d717c19ebde386b25e51dd546413e7ee997c5e1",
+ "1c4a73588541a0c869b4ee27c32cc4218f3c8db13713c61cedc387336a2720c9",
+ "1c73442f0ac53d8b38f779231680fab806a6cb9c86d25d9db5fa67c0ebf8e803",
+ "19152f0c06baf7962ca287a303b85437f321d725985f1586ac8358bdb6a0df63",
+ "13436f337815f5929559e6f621b850ed60b36f23ce9d8d06db981b70d40ad3db",
+ "298268d866eea5d6fcae470fdbb6d7787d73ab1e50b8126d6452d81264fbdafd",
+ "1a67b4e4c213baa140c5a00352cdbc9256b4e2fe81482c380b08ebe2e6b76e1b",
+ "19824d2008be54e35a0e2a9d2df9746e96f73367518b111695e1c3857966c241",
+ "2997c49ed21482d30b8ef89bd26bfdb6384dda6825032145fe0a3ad9d2f2a7e3",
+ "137ccc1d4ab00210bd9af5ee875cb799bd818f4803470abca68a9655ea73be01",
+ "12d4cf2eb41c90ede84bece72f76e97d7d0144c45341a0176f656b295cb838c3",
+ "11d02da4f449e6aeee4f213409baed6eaab35496688d743991870ba093676c44",
+ "163fb1ef04a1453a44fb897014287f7ceefe0b553d99718f986ada32cec6ca29",
+ "16f579a7eda4d7f5cde29724bf35e1b36e95fbeb72914ba2ba8f19192b92dab7",
+ "0f60c1387bf29d8d81174bd43c136e75f0f33b8b4d2712de0cc3a23f04fac76e",
+ "0f83662d3b4cc9aaa0f76c8801d2d32909c050286d514acc108b6c3d9964679b",
+ "1a30b7c4bf1c4eaaa92802cb90a27b5073d4a5ec095271490df8443b8f5df88f",
+ "105af591b517f126c985f54e833d61907ff49945ab123a606caa6d9bda0e3d66",
+ "1a5196fdfc1b81974905a66e6f1ff31403fc84b4d84effde521e848153f42e10",
+ "17d6cb5ba9489d8397cb1e1d054e45cff6c7225aeeba5c9e76cacd9da6c9a0c1",
+ "127ab4e2169329082bdd546e94c4fb6643999b14a26e08eaa719689789767014",
+ "143883410f000b5f3ff4c6434b0654516e9502d0a50a2b3ecdc20c8d3e198915",
+ "16ccd345646dd3d39e6bd157b51513c1b319bd1f441199003052a74b2eddb53d",
+ "11e9f02dc56d575fac5a306a6e83f597ffda1bd81a01f13fdda059ab24d90892",
+ "13f75a9e662faea5fc93f0f83d363c42083345cdcc42f1b0d320d11409ef3052",
+ "18cca97e8c6ede52e0b7d8c53c85c0fac68f6d1b7c6622a4cebc21433e6d8eea",
+ "160d6b818fab5ad00a1e81e46888c5ff3e5f2c175c013ce17d88c31df4475aba",
+ "1c4d09dff19175af7fc0d8e8fd23e9288fc2839dedfc067dcf9f5a3e3a9d92aa",
+ "16e25b2a6eef4cde6879c20c94c4360604b9099c29e1abaf9fc079fe67cfcaac",
+ "2a577ab7e2541e2cc2cc20e6a76c4ea9b77501808db9c4045be82c680cf227d5",
+ "11b4753fd9cc33656dbd59769b3202b7f68bd067bf7f64bd54676f6f60366ef1",
+ "1932a0aecc4a569d7d3fbcdd329b92c0b4dbd870d6be48ec4f18285ab3183676",
+ "2a2e6b62a4383cb48ffbb69b2f356ceb0410593f5b5500142498692dec7c125f",
+};
+
+
+Y_UNIT_TEST_SUITE(TMiniKQLToDictTest) {
+ Y_UNIT_TEST_LLVM(TestCompactUtf8Set) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TVector<TRuntimeNode> items;
+ for (auto s: data) {
+ items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s));
+ }
+ Shuffle(items.begin(), items.end());
+ auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ auto list = pb.NewList(dataType, items);
+ auto dict = pb.ToHashedDict(list, false, [](TRuntimeNode n) { return n; }, [&pb](TRuntimeNode /*n*/) { return pb.NewVoid(); }, true);
+ auto pgmReturn = pb.Contains(dict, items.front());
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto res = graph->GetValue().template Get<bool>();
+ UNIT_ASSERT_VALUES_EQUAL(res, true);
+ }
+
+ Y_UNIT_TEST_LLVM(TestUtf8Set) {
+ TSetup<LLVM> setup;
+ TProgramBuilder& pb = *setup.PgmBuilder;
+
+ TVector<TRuntimeNode> items;
+ for (auto s: data) {
+ items.push_back(pb.NewDataLiteral<NUdf::EDataSlot::Utf8>(s));
+ }
+ Shuffle(items.begin(), items.end());
+ auto dataType = pb.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
+ auto list = pb.NewList(dataType, items);
+ auto dict = pb.ToHashedDict(list, false, [](TRuntimeNode n) { return n; }, [&pb](TRuntimeNode /*n*/) { return pb.NewVoid(); }, false);
+ auto pgmReturn = pb.Contains(dict, items.front());
+
+ auto graph = setup.BuildGraph(pgmReturn);
+ auto res = graph->GetValue().template Get<bool>();
+ UNIT_ASSERT_VALUES_EQUAL(res, true);
+ }
Y_UNIT_TEST_LLVM(TestSqueezeToDict) {
auto test = [](bool stream, bool hashed, bool multi, bool compact, bool withPayload) {
@@ -214,7 +214,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLToDictTest) {
}
}
#endif
-}
-
-}
-}
+}
+
+}
+}
diff --git a/ydb/library/yql/minikql/comp_nodes/ut/ya.make b/ydb/library/yql/minikql/comp_nodes/ut/ya.make
index dffda1318f..16e6fcbf97 100644
--- a/ydb/library/yql/minikql/comp_nodes/ut/ya.make
+++ b/ydb/library/yql/minikql/comp_nodes/ut/ya.make
@@ -20,7 +20,7 @@ OWNER(
SRCS(
mkql_blocks_ut.cpp
- mkql_combine_ut.cpp
+ mkql_combine_ut.cpp
mkql_condense_ut.cpp
mkql_decimal_ut.cpp
mkql_chain_map_ut.cpp
@@ -34,7 +34,7 @@ SRCS(
mkql_heap_ut.cpp
mkql_compare_ut.cpp
mkql_computation_node_ut.cpp
- mkql_group_ut.cpp
+ mkql_group_ut.cpp
mkql_dict_ut.cpp
mkql_join_ut.cpp
mkql_join_dict_ut.cpp
@@ -42,7 +42,7 @@ SRCS(
mkql_safe_circular_buffer_ut.cpp
mkql_sort_ut.cpp
mkql_switch_ut.cpp
- mkql_todict_ut.cpp
+ mkql_todict_ut.cpp
mkql_variant_ut.cpp
mkql_wide_chain_map_ut.cpp
mkql_wide_chopper_ut.cpp
diff --git a/ydb/library/yql/minikql/comp_nodes/ya.make b/ydb/library/yql/minikql/comp_nodes/ya.make
index 55838c4012..f65cda7c4c 100644
--- a/ydb/library/yql/minikql/comp_nodes/ya.make
+++ b/ydb/library/yql/minikql/comp_nodes/ya.make
@@ -32,8 +32,8 @@ SRCS(
mkql_coalesce.h
mkql_collect.cpp
mkql_collect.h
- mkql_combine.cpp
- mkql_combine.h
+ mkql_combine.cpp
+ mkql_combine.h
mkql_contains.cpp
mkql_contains.h
mkql_decimal_div.cpp
@@ -74,8 +74,8 @@ SRCS(
mkql_fromstring.h
mkql_fromyson.cpp
mkql_fromyson.h
- mkql_group.cpp
- mkql_group.h
+ mkql_group.cpp
+ mkql_group.h
mkql_guess.cpp
mkql_guess.h
mkql_hasitems.cpp
@@ -183,8 +183,8 @@ SRCS(
mkql_udf.h
mkql_unwrap.cpp
mkql_unwrap.h
- mkql_varitem.cpp
- mkql_varitem.h
+ mkql_varitem.cpp
+ mkql_varitem.h
mkql_visitall.cpp
mkql_visitall.h
mkql_way.cpp
diff --git a/ydb/library/yql/minikql/compact_hash.cpp b/ydb/library/yql/minikql/compact_hash.cpp
index 17810ccacb..9e49417385 100644
--- a/ydb/library/yql/minikql/compact_hash.cpp
+++ b/ydb/library/yql/minikql/compact_hash.cpp
@@ -1,63 +1,63 @@
-#include "compact_hash.h"
-
-#include <util/generic/map.h>
-#include <util/stream/str.h>
-
-
-namespace NKikimr {
-
-namespace NCHash {
-
-void TListPoolBase::FreeListPage(TListHeader* p) {
- Y_ASSERT(TAlignedPagePool::GetPageStart(p) == p);
- p->~TListHeader();
+#include "compact_hash.h"
+
+#include <util/generic/map.h>
+#include <util/stream/str.h>
+
+
+namespace NKikimr {
+
+namespace NCHash {
+
+void TListPoolBase::FreeListPage(TListHeader* p) {
+ Y_ASSERT(TAlignedPagePool::GetPageStart(p) == p);
+ p->~TListHeader();
PagePool_.ReturnPage(p);
-}
-
+}
+
size_t TListPoolBase::TUsedPages::PrintStat(const TStringBuf& header, IOutputStream& out) const {
TMap<ui32, ui64> counts;
- size_t pages = 0;
- for (auto& p: FullPages) {
- ++pages;
- ++counts[p.As<TListHeader>()->ListSize];
- }
- for (auto& c: counts) {
- out << header << "Count of full pages<" << c.first << ">: " << c.second << Endl;
- }
- for (size_t i = 0; i < SmallPages.size(); ++i) {
- if (size_t size = SmallPages[i].Size()) {
- pages += size;
- out << header << "Count of partially free pages<" << SmallPages[i].Front()->As<TListHeader>()->ListSize << ">: " << size << Endl;
- }
- }
- for (size_t i = 0; i < MediumPages.size(); ++i) {
- if (size_t size = MediumPages[i].Size()) {
- pages += size;
- out << header << "Count of partially free pages<" << MediumPages[i].Front()->As<TListHeader>()->ListSize << ">: " << size << Endl;
- }
- }
- return pages;
-}
-
-TString TListPoolBase::TUsedPages::DebugInfo() const {
- TStringStream out;
- TMap<ui32, ui64> counts;
- for (auto& p: FullPages) {
- out << "Full page<" << p.As<TListHeader>()->ListSize << ">: " << p.As<TListHeader>()->FreeLists << Endl;
- }
- for (size_t i = 0; i < SmallPages.size(); ++i) {
- for (auto& p: SmallPages[i]) {
- out << "Partially free page<" << p.As<TListHeader>()->ListSize << ">: " << p.As<TListHeader>()->FreeLists << Endl;
- }
- }
- for (size_t i = 0; i < MediumPages.size(); ++i) {
- for (auto& p: MediumPages[i]) {
- out << "Partially free page<" << p.As<TListHeader>()->ListSize << ">: " << p.As<TListHeader>()->FreeLists << Endl;
- }
- }
- return out.Str();
-}
-
-} // NCHash
-
-} // NKikimr
+ size_t pages = 0;
+ for (auto& p: FullPages) {
+ ++pages;
+ ++counts[p.As<TListHeader>()->ListSize];
+ }
+ for (auto& c: counts) {
+ out << header << "Count of full pages<" << c.first << ">: " << c.second << Endl;
+ }
+ for (size_t i = 0; i < SmallPages.size(); ++i) {
+ if (size_t size = SmallPages[i].Size()) {
+ pages += size;
+ out << header << "Count of partially free pages<" << SmallPages[i].Front()->As<TListHeader>()->ListSize << ">: " << size << Endl;
+ }
+ }
+ for (size_t i = 0; i < MediumPages.size(); ++i) {
+ if (size_t size = MediumPages[i].Size()) {
+ pages += size;
+ out << header << "Count of partially free pages<" << MediumPages[i].Front()->As<TListHeader>()->ListSize << ">: " << size << Endl;
+ }
+ }
+ return pages;
+}
+
+TString TListPoolBase::TUsedPages::DebugInfo() const {
+ TStringStream out;
+ TMap<ui32, ui64> counts;
+ for (auto& p: FullPages) {
+ out << "Full page<" << p.As<TListHeader>()->ListSize << ">: " << p.As<TListHeader>()->FreeLists << Endl;
+ }
+ for (size_t i = 0; i < SmallPages.size(); ++i) {
+ for (auto& p: SmallPages[i]) {
+ out << "Partially free page<" << p.As<TListHeader>()->ListSize << ">: " << p.As<TListHeader>()->FreeLists << Endl;
+ }
+ }
+ for (size_t i = 0; i < MediumPages.size(); ++i) {
+ for (auto& p: MediumPages[i]) {
+ out << "Partially free page<" << p.As<TListHeader>()->ListSize << ">: " << p.As<TListHeader>()->FreeLists << Endl;
+ }
+ }
+ return out.Str();
+}
+
+} // NCHash
+
+} // NKikimr
diff --git a/ydb/library/yql/minikql/compact_hash.h b/ydb/library/yql/minikql/compact_hash.h
index a9586db464..9c59903c89 100644
--- a/ydb/library/yql/minikql/compact_hash.h
+++ b/ydb/library/yql/minikql/compact_hash.h
@@ -1,292 +1,292 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/utils/hash.h>
-#include "aligned_page_pool.h"
-#include "primes.h"
-
-#include <util/generic/vector.h>
-#include <util/generic/ptr.h>
-#include <util/generic/hash.h>
-#include <util/generic/algorithm.h>
-#include <util/generic/bitops.h>
-#include <util/generic/utility.h>
-#include <util/generic/strbuf.h>
-#include <util/generic/mem_copy.h>
-#include <util/generic/scope.h>
-#include <util/system/defaults.h>
-#include <util/system/unaligned_mem.h>
-#include <util/system/align.h>
+#include "aligned_page_pool.h"
+#include "primes.h"
+
+#include <util/generic/vector.h>
+#include <util/generic/ptr.h>
+#include <util/generic/hash.h>
+#include <util/generic/algorithm.h>
+#include <util/generic/bitops.h>
+#include <util/generic/utility.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/mem_copy.h>
+#include <util/generic/scope.h>
+#include <util/system/defaults.h>
+#include <util/system/unaligned_mem.h>
+#include <util/system/align.h>
#include <util/stream/output.h>
-
-#include <type_traits>
-#include <cmath>
-#include <array>
-
-// TODO: Allow only PODs in key/value. Use runtime key/value sizes instead of compile time type instantiation. Use Read/WriteUnaligned
-
-namespace NKikimr {
-
-namespace NCHash {
-
+
+#include <type_traits>
+#include <cmath>
+#include <array>
+
+// TODO: Allow only PODs in key/value. Use runtime key/value sizes instead of compile time type instantiation. Use Read/WriteUnaligned
+
+namespace NKikimr {
+
+namespace NCHash {
+
class TListPoolBase {
-public:
- enum {
- SMALL_MARK = 1,
- MEDIUM_MARK,
- LARGE_MARK,
- };
- static const size_t MAX_SMALL_LIST_SIZE = 16;
- static const size_t MAX_MEDIUM_LIST_INDEX = 10;
-
-public:
- struct TPageListItem: public TIntrusiveListItem<TPageListItem> {
- template <class T>
- T* As() {
- return static_cast<T*>(TAlignedPagePool::GetPageStart(this));
- }
- template <class T>
- const T* As() const {
- return static_cast<const T*>(TAlignedPagePool::GetPageStart(this));
- }
- };
-
- struct TListHeader {
- ui16 Mark;
- ui16 ListSize;
- ui16 FreeLists;
- ui16 FreeListOffset;
- TPageListItem ListItem;
-
- TListHeader(ui16 mark, ui16 listSize, ui16 listCount)
- : Mark(mark)
- , ListSize(listSize)
- , FreeLists(listCount)
- , FreeListOffset(0u)
- {
- Y_ASSERT(ListItem.As<TListHeader>() == this);
- *reinterpret_cast<ui16*>(this + 1) = 0u; // Mark first list for initial usage
- }
- ~TListHeader() {
- Mark = 0u; // Reset mark
- }
- };
-
- struct TLargeListHeader {
- ui16 Mark;
- ui32 Capacity;
- ui32 Size;
- TPageListItem ListItem;
-
- TLargeListHeader(ui32 capacity)
- : Mark(LARGE_MARK)
- , Capacity(capacity)
- , Size(0u)
- {
- Y_ASSERT(ListItem.As<TLargeListHeader>() == this);
- }
- ~TLargeListHeader() {
- Mark = 0u; // Reset mark
- }
- template <typename T>
- T* GetList() {
- return reinterpret_cast<T*>(this + 1);
- }
-
- template <typename T>
- const T* GetList() const {
- return reinterpret_cast<const T*>(this + 1);
- }
-
- TLargeListHeader* Next() {
- return ListItem.Next()->Node()->As<TLargeListHeader>();
- }
-
- const TLargeListHeader* Next() const {
- return ListItem.Next()->Node()->As<TLargeListHeader>();
- }
-
- void Append(TLargeListHeader* header) {
- header->ListItem.LinkAfter(ListItem);
- }
- };
-
- template <typename T, class THeader>
- class TListIterator {
- using TRawByte = std::conditional_t<std::is_const<T>::value, const ui8*, ui8*>;
- public:
- using TRaw = std::conditional_t<std::is_const<T>::value, const void*, void*>;
-
- TListIterator() {
- }
-
- TListIterator(T* list) {
- if (LARGE_MARK == GetMark(list)) {
- CurrentPage = EndPage = GetLargeListHeader(list)->Next();
- Current = CurrentPage->template GetList<T>();
- End = (TRawByte)Current + CurrentPage->Size * sizeof(T);
- } else {
- Current = list;
- End = (TRawByte)Current + GetPartListSize(list) * sizeof(T);
- }
- }
-
- TListIterator(TRaw start, TRaw end)
- : Current(start)
- , End(end)
- {
- }
-
- TListIterator& operator++() {
- Y_ASSERT(Ok());
- Current = (TRawByte)Current + sizeof(T);
- if (Current == End) {
- if (CurrentPage) {
- CurrentPage = CurrentPage->Next();
- if (CurrentPage != EndPage) {
- Current = (TRaw)CurrentPage->template GetList<T>();
- End = (TRawByte)Current + CurrentPage->Size * sizeof(T);
- } else {
- CurrentPage = EndPage = nullptr;
- Current = End = nullptr;
- }
- } else {
- Current = End = nullptr;
- }
- }
- return *this;
- }
-
- bool operator ==(const TListIterator& it) const {
- return Current == it.Current && CurrentPage == it.CurrentPage;
- }
-
- bool operator !=(const TListIterator& it) const {
- return !operator ==(it);
- }
-
- T Get() const {
- using TClean = typename std::remove_cv<T>::type;
- return ::ReadUnaligned<TClean>(Current);
- }
-
- void Set(T val) const {
- ::WriteUnaligned<T>(Current, val);
- }
-
- TRaw GetRaw() const {
- return Current;
- }
-
- bool Ok() const {
- return Current != nullptr;
- }
-
- private:
- TRaw Current = nullptr;
- TRaw End = nullptr;
- THeader* CurrentPage = nullptr;
- THeader* EndPage = nullptr;
- };
-
-public:
- static inline TListHeader* GetListHeader(void* addr) {
- TListHeader* header = reinterpret_cast<TListHeader*>(TAlignedPagePool::GetPageStart(addr));
- Y_ASSERT(SMALL_MARK == header->Mark || MEDIUM_MARK == header->Mark);
- return header;
- }
-
- static inline const TListHeader* GetListHeader(const void* addr) {
- const TListHeader* header = reinterpret_cast<const TListHeader*>(TAlignedPagePool::GetPageStart(addr));
- Y_ASSERT(SMALL_MARK == header->Mark || MEDIUM_MARK == header->Mark);
- return header;
- }
-
- static inline TLargeListHeader* GetLargeListHeader(void* addr) {
- TLargeListHeader* header = reinterpret_cast<TLargeListHeader*>(TAlignedPagePool::GetPageStart(addr));
- Y_ASSERT(LARGE_MARK == header->Mark);
- return header;
- }
-
- static inline const TLargeListHeader* GetLargeListHeader(const void* addr) {
- const TLargeListHeader* header = reinterpret_cast<const TLargeListHeader*>(TAlignedPagePool::GetPageStart(addr));
- Y_ASSERT(LARGE_MARK == header->Mark);
- return header;
- }
-
- template <typename T>
- static inline constexpr ui16 GetSmallPageCapacity(size_t listSize) {
- // Always keep at least ui16 per list in order to track collection free lists
- return static_cast<ui16>((TAlignedPagePool::POOL_PAGE_SIZE - sizeof(TListHeader)) / (AlignUp<size_t>(listSize * sizeof(T), sizeof(ui16)))) & 0x7FFF;
- }
-
- template <typename T>
- static inline constexpr ui16 GetMediumPageCapacity(size_t listSize) {
+public:
+ enum {
+ SMALL_MARK = 1,
+ MEDIUM_MARK,
+ LARGE_MARK,
+ };
+ static const size_t MAX_SMALL_LIST_SIZE = 16;
+ static const size_t MAX_MEDIUM_LIST_INDEX = 10;
+
+public:
+ struct TPageListItem: public TIntrusiveListItem<TPageListItem> {
+ template <class T>
+ T* As() {
+ return static_cast<T*>(TAlignedPagePool::GetPageStart(this));
+ }
+ template <class T>
+ const T* As() const {
+ return static_cast<const T*>(TAlignedPagePool::GetPageStart(this));
+ }
+ };
+
+ struct TListHeader {
+ ui16 Mark;
+ ui16 ListSize;
+ ui16 FreeLists;
+ ui16 FreeListOffset;
+ TPageListItem ListItem;
+
+ TListHeader(ui16 mark, ui16 listSize, ui16 listCount)
+ : Mark(mark)
+ , ListSize(listSize)
+ , FreeLists(listCount)
+ , FreeListOffset(0u)
+ {
+ Y_ASSERT(ListItem.As<TListHeader>() == this);
+ *reinterpret_cast<ui16*>(this + 1) = 0u; // Mark first list for initial usage
+ }
+ ~TListHeader() {
+ Mark = 0u; // Reset mark
+ }
+ };
+
+ struct TLargeListHeader {
+ ui16 Mark;
+ ui32 Capacity;
+ ui32 Size;
+ TPageListItem ListItem;
+
+ TLargeListHeader(ui32 capacity)
+ : Mark(LARGE_MARK)
+ , Capacity(capacity)
+ , Size(0u)
+ {
+ Y_ASSERT(ListItem.As<TLargeListHeader>() == this);
+ }
+ ~TLargeListHeader() {
+ Mark = 0u; // Reset mark
+ }
+ template <typename T>
+ T* GetList() {
+ return reinterpret_cast<T*>(this + 1);
+ }
+
+ template <typename T>
+ const T* GetList() const {
+ return reinterpret_cast<const T*>(this + 1);
+ }
+
+ TLargeListHeader* Next() {
+ return ListItem.Next()->Node()->As<TLargeListHeader>();
+ }
+
+ const TLargeListHeader* Next() const {
+ return ListItem.Next()->Node()->As<TLargeListHeader>();
+ }
+
+ void Append(TLargeListHeader* header) {
+ header->ListItem.LinkAfter(ListItem);
+ }
+ };
+
+ template <typename T, class THeader>
+ class TListIterator {
+ using TRawByte = std::conditional_t<std::is_const<T>::value, const ui8*, ui8*>;
+ public:
+ using TRaw = std::conditional_t<std::is_const<T>::value, const void*, void*>;
+
+ TListIterator() {
+ }
+
+ TListIterator(T* list) {
+ if (LARGE_MARK == GetMark(list)) {
+ CurrentPage = EndPage = GetLargeListHeader(list)->Next();
+ Current = CurrentPage->template GetList<T>();
+ End = (TRawByte)Current + CurrentPage->Size * sizeof(T);
+ } else {
+ Current = list;
+ End = (TRawByte)Current + GetPartListSize(list) * sizeof(T);
+ }
+ }
+
+ TListIterator(TRaw start, TRaw end)
+ : Current(start)
+ , End(end)
+ {
+ }
+
+ TListIterator& operator++() {
+ Y_ASSERT(Ok());
+ Current = (TRawByte)Current + sizeof(T);
+ if (Current == End) {
+ if (CurrentPage) {
+ CurrentPage = CurrentPage->Next();
+ if (CurrentPage != EndPage) {
+ Current = (TRaw)CurrentPage->template GetList<T>();
+ End = (TRawByte)Current + CurrentPage->Size * sizeof(T);
+ } else {
+ CurrentPage = EndPage = nullptr;
+ Current = End = nullptr;
+ }
+ } else {
+ Current = End = nullptr;
+ }
+ }
+ return *this;
+ }
+
+ bool operator ==(const TListIterator& it) const {
+ return Current == it.Current && CurrentPage == it.CurrentPage;
+ }
+
+ bool operator !=(const TListIterator& it) const {
+ return !operator ==(it);
+ }
+
+ T Get() const {
+ using TClean = typename std::remove_cv<T>::type;
+ return ::ReadUnaligned<TClean>(Current);
+ }
+
+ void Set(T val) const {
+ ::WriteUnaligned<T>(Current, val);
+ }
+
+ TRaw GetRaw() const {
+ return Current;
+ }
+
+ bool Ok() const {
+ return Current != nullptr;
+ }
+
+ private:
+ TRaw Current = nullptr;
+ TRaw End = nullptr;
+ THeader* CurrentPage = nullptr;
+ THeader* EndPage = nullptr;
+ };
+
+public:
+ static inline TListHeader* GetListHeader(void* addr) {
+ TListHeader* header = reinterpret_cast<TListHeader*>(TAlignedPagePool::GetPageStart(addr));
+ Y_ASSERT(SMALL_MARK == header->Mark || MEDIUM_MARK == header->Mark);
+ return header;
+ }
+
+ static inline const TListHeader* GetListHeader(const void* addr) {
+ const TListHeader* header = reinterpret_cast<const TListHeader*>(TAlignedPagePool::GetPageStart(addr));
+ Y_ASSERT(SMALL_MARK == header->Mark || MEDIUM_MARK == header->Mark);
+ return header;
+ }
+
+ static inline TLargeListHeader* GetLargeListHeader(void* addr) {
+ TLargeListHeader* header = reinterpret_cast<TLargeListHeader*>(TAlignedPagePool::GetPageStart(addr));
+ Y_ASSERT(LARGE_MARK == header->Mark);
+ return header;
+ }
+
+ static inline const TLargeListHeader* GetLargeListHeader(const void* addr) {
+ const TLargeListHeader* header = reinterpret_cast<const TLargeListHeader*>(TAlignedPagePool::GetPageStart(addr));
+ Y_ASSERT(LARGE_MARK == header->Mark);
+ return header;
+ }
+
+ template <typename T>
+ static inline constexpr ui16 GetSmallPageCapacity(size_t listSize) {
+ // Always keep at least ui16 per list in order to track collection free lists
+ return static_cast<ui16>((TAlignedPagePool::POOL_PAGE_SIZE - sizeof(TListHeader)) / (AlignUp<size_t>(listSize * sizeof(T), sizeof(ui16)))) & 0x7FFF;
+ }
+
+ template <typename T>
+ static inline constexpr ui16 GetMediumPageCapacity(size_t listSize) {
return static_cast<ui16>((TAlignedPagePool::POOL_PAGE_SIZE - sizeof(TListHeader)) / (FastClp2(listSize) * sizeof(T) + sizeof(ui16))) & 0x7FFF;
- }
-
- template <typename T>
- static inline constexpr ui32 GetLargePageCapacity() {
+ }
+
+ template <typename T>
+ static inline constexpr ui32 GetLargePageCapacity() {
return static_cast<ui32>((TAlignedPagePool::POOL_PAGE_SIZE - sizeof(TLargeListHeader)) / sizeof(T));
- }
-
- template <typename T>
- static inline constexpr size_t GetMaxListSize() {
+ }
+
+ template <typename T>
+ static inline constexpr size_t GetMaxListSize() {
#define NCHASH_RAW_SIZE ((((TAlignedPagePool::POOL_PAGE_SIZE - sizeof(TListHeader)) / 2) - sizeof(ui16)) / sizeof(T))
- // For medium lists get downward pow2 num
- return NCHASH_RAW_SIZE > 16 ? 1ULL << MostSignificantBitCT(NCHASH_RAW_SIZE) : NCHASH_RAW_SIZE;
-#undef NCHASH_RAW_SIZE
- }
-
-protected:
+ // For medium lists get downward pow2 num
+ return NCHASH_RAW_SIZE > 16 ? 1ULL << MostSignificantBitCT(NCHASH_RAW_SIZE) : NCHASH_RAW_SIZE;
+#undef NCHASH_RAW_SIZE
+ }
+
+protected:
using TListType = TIntrusiveList<TPageListItem>;
- struct TUsedPages {
- TUsedPages() = default;
- TUsedPages(const TUsedPages&) = delete;
- TUsedPages(TUsedPages&& other)
- : SmallPages(std::move(other.SmallPages))
- , MediumPages(std::move(other.MediumPages))
- , FullPages(std::move(other.FullPages))
- {
- }
-
- TUsedPages& operator =(const TUsedPages& other) = delete;
- TUsedPages& operator =(TUsedPages&& other) {
- TUsedPages(std::move(other)).Swap(*this);
- return *this;
- }
-
- void Swap(TUsedPages& other) {
- DoSwap(SmallPages, other.SmallPages);
- DoSwap(MediumPages, other.MediumPages);
- DoSwap(FullPages, other.FullPages);
- }
-
- // Returns count of used pages
+ struct TUsedPages {
+ TUsedPages() = default;
+ TUsedPages(const TUsedPages&) = delete;
+ TUsedPages(TUsedPages&& other)
+ : SmallPages(std::move(other.SmallPages))
+ , MediumPages(std::move(other.MediumPages))
+ , FullPages(std::move(other.FullPages))
+ {
+ }
+
+ TUsedPages& operator =(const TUsedPages& other) = delete;
+ TUsedPages& operator =(TUsedPages&& other) {
+ TUsedPages(std::move(other)).Swap(*this);
+ return *this;
+ }
+
+ void Swap(TUsedPages& other) {
+ DoSwap(SmallPages, other.SmallPages);
+ DoSwap(MediumPages, other.MediumPages);
+ DoSwap(FullPages, other.FullPages);
+ }
+
+ // Returns count of used pages
size_t PrintStat(const TStringBuf& header, IOutputStream& out) const;
- TString DebugInfo() const;
-
+ TString DebugInfo() const;
+
std::array<TListType, MAX_SMALL_LIST_SIZE - 1> SmallPages; // 2-16 sizes
std::array<TListType, MAX_MEDIUM_LIST_INDEX> MediumPages; // 32,64,128,...,16384. Indexed by pow2
TListType FullPages;
- };
-
-public:
+ };
+
+public:
TListPoolBase(TAlignedPagePool& pagePool)
: PagePool_(pagePool)
{}
- TListPoolBase(const TListPoolBase&) = delete;
- TListPoolBase(TListPoolBase&& other)
+ TListPoolBase(const TListPoolBase&) = delete;
+ TListPoolBase(TListPoolBase&& other)
: PagePool_(other.PagePool_)
- {
- }
-
- TListPoolBase& operator =(const TListPoolBase& other) = delete;
- TListPoolBase& operator =(TListPoolBase&& other) {
+ {
+ }
+
+ TListPoolBase& operator =(const TListPoolBase& other) = delete;
+ TListPoolBase& operator =(TListPoolBase&& other) {
PagePool_.Swap(other.PagePool_);
- return *this;
- }
-
+ return *this;
+ }
+
void Swap(TListPoolBase& other) {
PagePool_.Swap(other.PagePool_);
}
@@ -295,1017 +295,1017 @@ public:
return PagePool_;
}
- // Returns full list size for short and medium lists. Returns size of last page for a large list
- static size_t GetPartListSize(const void* list) {
- switch (GetMark(list)) {
- case SMALL_MARK:
- return GetListHeader(list)->ListSize;
- case MEDIUM_MARK:
- return *(reinterpret_cast<const ui16*>(list) - 1);
- case LARGE_MARK:
- return GetLargeListHeader(list)->Size;
- default:
- Y_FAIL("Bad list address");
- }
- return 0;
- }
-
- static size_t GetFullListSize(const void* list) {
- switch (GetMark(list)) {
- case SMALL_MARK:
- return GetListHeader(list)->ListSize;
- case MEDIUM_MARK:
- return *(reinterpret_cast<const ui16*>(list) - 1);
- case LARGE_MARK:
- {
- const TLargeListHeader* start = GetLargeListHeader(list);
- const TLargeListHeader* next = start;
- size_t size = 0;
- do {
- size += next->Size;
- next = next->Next();
- } while (next != start);
- return size;
- }
- default:
- Y_FAIL("Bad list address");
- }
- return 0;
- }
-
- static size_t GetListCapacity(const void* list) {
- switch (GetMark(list)) {
- case SMALL_MARK:
- case MEDIUM_MARK:
- return GetListHeader(list)->ListSize;
- case LARGE_MARK:
- return GetLargeListHeader(list)->Capacity;
- default:
- Y_FAIL("Bad list address");
- }
- return 0;
- }
-
- static void SetPartListSize(void* list, size_t size) {
- switch (GetMark(list)) {
- case MEDIUM_MARK:
- *(reinterpret_cast<ui16*>(list) - 1) = size;
- break;
- case LARGE_MARK:
- GetLargeListHeader(list)->Size = size;
- break;
- default:
- Y_FAIL("Bad list address");
- }
- }
-
- static inline ui16 GetMark(const void* addr) {
- return *reinterpret_cast<const ui16*>(TAlignedPagePool::GetPageStart(addr));
- }
-
-protected:
- void FreeListPage(TListHeader* p);
+ // Returns full list size for short and medium lists. Returns size of last page for a large list
+ static size_t GetPartListSize(const void* list) {
+ switch (GetMark(list)) {
+ case SMALL_MARK:
+ return GetListHeader(list)->ListSize;
+ case MEDIUM_MARK:
+ return *(reinterpret_cast<const ui16*>(list) - 1);
+ case LARGE_MARK:
+ return GetLargeListHeader(list)->Size;
+ default:
+ Y_FAIL("Bad list address");
+ }
+ return 0;
+ }
+
+ static size_t GetFullListSize(const void* list) {
+ switch (GetMark(list)) {
+ case SMALL_MARK:
+ return GetListHeader(list)->ListSize;
+ case MEDIUM_MARK:
+ return *(reinterpret_cast<const ui16*>(list) - 1);
+ case LARGE_MARK:
+ {
+ const TLargeListHeader* start = GetLargeListHeader(list);
+ const TLargeListHeader* next = start;
+ size_t size = 0;
+ do {
+ size += next->Size;
+ next = next->Next();
+ } while (next != start);
+ return size;
+ }
+ default:
+ Y_FAIL("Bad list address");
+ }
+ return 0;
+ }
+
+ static size_t GetListCapacity(const void* list) {
+ switch (GetMark(list)) {
+ case SMALL_MARK:
+ case MEDIUM_MARK:
+ return GetListHeader(list)->ListSize;
+ case LARGE_MARK:
+ return GetLargeListHeader(list)->Capacity;
+ default:
+ Y_FAIL("Bad list address");
+ }
+ return 0;
+ }
+
+ static void SetPartListSize(void* list, size_t size) {
+ switch (GetMark(list)) {
+ case MEDIUM_MARK:
+ *(reinterpret_cast<ui16*>(list) - 1) = size;
+ break;
+ case LARGE_MARK:
+ GetLargeListHeader(list)->Size = size;
+ break;
+ default:
+ Y_FAIL("Bad list address");
+ }
+ }
+
+ static inline ui16 GetMark(const void* addr) {
+ return *reinterpret_cast<const ui16*>(TAlignedPagePool::GetPageStart(addr));
+ }
+
+protected:
+ void FreeListPage(TListHeader* p);
private:
TAlignedPagePool& PagePool_;
-};
-
-template <typename TPrimary, typename TSecondary = TPrimary>
-class TListPool: public TListPoolBase {
-private:
- static constexpr size_t PoolCount = 1 + !std::is_same<TPrimary, TSecondary>::value;
-
-public:
+};
+
+template <typename TPrimary, typename TSecondary = TPrimary>
+class TListPool: public TListPoolBase {
+private:
+ static constexpr size_t PoolCount = 1 + !std::is_same<TPrimary, TSecondary>::value;
+
+public:
TListPool(TAlignedPagePool& pagePool)
: TListPoolBase(pagePool)
{}
- TListPool(const TListPool&) = delete;
- TListPool(TListPool&& other)
- : TListPoolBase(std::move(other))
- , Pools(std::move(other.Pools))
- {
- }
-
- ~TListPool() {
- for (auto& p: Pools) {
- for (auto& list: p.SmallPages) {
- Y_VERIFY(list.Empty(), "%s", DebugInfo().data());
- }
- for (auto& list: p.MediumPages) {
- Y_VERIFY(list.Empty(), "%s", DebugInfo().data());
- }
- Y_VERIFY(p.FullPages.Empty(), "%s", DebugInfo().data());
- }
- }
-
- TListPool& operator =(const TListPool& other) = delete;
- TListPool& operator =(TListPool&& other) {
- TListPool(std::move(other)).Swap(*this);
- return *this;
- }
-
- template <typename T>
- T* GetList(size_t size) {
- static_assert(std::is_same<TPrimary, T>::value || std::is_same<TSecondary, T>::value, "Bad requested list type");
- static constexpr size_t PoolNdx = static_cast<size_t>(!std::is_same<TPrimary, T>::value);
- Y_VERIFY(size >= 2);
- T* res = nullptr;
-
- if (Y_LIKELY(size <= TListPoolBase::GetMaxListSize<T>())) {
- if (Y_LIKELY(size <= MAX_SMALL_LIST_SIZE)) {
- res = GetSmallList<PoolNdx, T>(GetSmallListPage<PoolNdx, T>(size));
- } else {
- res = GetMediumList<PoolNdx, T>(GetMediumListPage<PoolNdx, T>(size), size);
- }
- }
-
- return res;
- }
-
- // Returns pointer to the new element
- template <typename T>
- T* IncrementList(T*& list) {
- size_t oldSize = TListPoolBase::GetPartListSize(list);
- if (oldSize + 1 <= TListPoolBase::GetListCapacity(list)) {
- TListPoolBase::SetPartListSize(list, oldSize + 1);
- return list + oldSize;
- }
-
- TLargeListHeader* header = nullptr;
- T* oldList = list;
- if (Y_LIKELY(LARGE_MARK != GetMark(oldList))) {
- list = GetList<T>(oldSize + 1);
- if (nullptr == list) {
- // Convert to large list
- header = GetLargeListPage<T>();
- list = header->GetList<T>();
- Y_VERIFY(header->Capacity >= oldSize);
- header->Size = oldSize;
- }
-
- if (std::is_trivially_copyable<T>::value) {
- memcpy(list, oldList, sizeof(T) * oldSize);
- } else {
- for (size_t i = 0; i < oldSize; ++i) {
- ::WriteUnaligned<T>(list + i, ::ReadUnaligned<T>(oldList + i));
- }
- }
- ReturnList(oldList);
-
- if (nullptr == header) {
- return list + oldSize;
- } else if (header->Size < header->Capacity) {
- return header->GetList<T>() + header->Size++;
- }
- } else {
- header = GetLargeListHeader(list);
- }
-
- // Add new page to large list
- TLargeListHeader* newPage = GetLargeListPage<T>();
- ++newPage->Size;
- list = newPage->GetList<T>();
- header->Append(newPage);
- return list;
- }
-
- template <typename T>
- T* CloneList(const T* list) {
- if (Y_LIKELY(LARGE_MARK != GetMark(list))) {
- const size_t size = TListPoolBase::GetPartListSize(list);
- T* clonedList = GetList<T>(size);
- for (size_t i = 0; i < size; ++i) {
- new (clonedList + i) T(list[i]);
- }
- return clonedList;
- } else {
- TLargeListHeader* lastListHeader = nullptr;
- const TLargeListHeader* start = GetLargeListHeader(list)->Next();
- const TLargeListHeader* header = start;
- do {
- TLargeListHeader* newHeader = GetLargeListPage<T>();
- newHeader->Size = header->Size;
- T* newList = newHeader->GetList<T>();
- const T* list = header->GetList<T>();
- for (size_t i = 0; i < header->Size; ++i) {
- new (newList + i) T(list[i]);
- }
- if (lastListHeader) {
- lastListHeader->Append(newHeader);
- }
- lastListHeader = newHeader;
- header = header->Next();
- } while (header != start);
-
- return lastListHeader->GetList<T>();
- }
- }
-
- template <typename T>
- void ReturnList(T* list) {
- static_assert(std::is_same<TPrimary, T>::value || std::is_same<TSecondary, T>::value, "Bad returned list type");
- static constexpr size_t PoolNdx = static_cast<size_t>(!std::is_same<TPrimary, T>::value);
- switch (GetMark(list)) {
- case SMALL_MARK:
- ReturnSmallList<PoolNdx, T>(GetListHeader(list), list);
- break;
- case MEDIUM_MARK:
- ReturnMediumList<PoolNdx, T>(GetListHeader(list), list);
- break;
- case LARGE_MARK:
- ReturnLargeList<T>(list);
- break;
- default:
- Y_FAIL("Bad list address");
- }
- }
-
- void Swap(TListPool& other) {
- TListPoolBase::Swap(other);
- DoSwap(Pools, other.Pools);
- }
-
+ TListPool(const TListPool&) = delete;
+ TListPool(TListPool&& other)
+ : TListPoolBase(std::move(other))
+ , Pools(std::move(other.Pools))
+ {
+ }
+
+ ~TListPool() {
+ for (auto& p: Pools) {
+ for (auto& list: p.SmallPages) {
+ Y_VERIFY(list.Empty(), "%s", DebugInfo().data());
+ }
+ for (auto& list: p.MediumPages) {
+ Y_VERIFY(list.Empty(), "%s", DebugInfo().data());
+ }
+ Y_VERIFY(p.FullPages.Empty(), "%s", DebugInfo().data());
+ }
+ }
+
+ TListPool& operator =(const TListPool& other) = delete;
+ TListPool& operator =(TListPool&& other) {
+ TListPool(std::move(other)).Swap(*this);
+ return *this;
+ }
+
+ template <typename T>
+ T* GetList(size_t size) {
+ static_assert(std::is_same<TPrimary, T>::value || std::is_same<TSecondary, T>::value, "Bad requested list type");
+ static constexpr size_t PoolNdx = static_cast<size_t>(!std::is_same<TPrimary, T>::value);
+ Y_VERIFY(size >= 2);
+ T* res = nullptr;
+
+ if (Y_LIKELY(size <= TListPoolBase::GetMaxListSize<T>())) {
+ if (Y_LIKELY(size <= MAX_SMALL_LIST_SIZE)) {
+ res = GetSmallList<PoolNdx, T>(GetSmallListPage<PoolNdx, T>(size));
+ } else {
+ res = GetMediumList<PoolNdx, T>(GetMediumListPage<PoolNdx, T>(size), size);
+ }
+ }
+
+ return res;
+ }
+
+ // Returns pointer to the new element
+ template <typename T>
+ T* IncrementList(T*& list) {
+ size_t oldSize = TListPoolBase::GetPartListSize(list);
+ if (oldSize + 1 <= TListPoolBase::GetListCapacity(list)) {
+ TListPoolBase::SetPartListSize(list, oldSize + 1);
+ return list + oldSize;
+ }
+
+ TLargeListHeader* header = nullptr;
+ T* oldList = list;
+ if (Y_LIKELY(LARGE_MARK != GetMark(oldList))) {
+ list = GetList<T>(oldSize + 1);
+ if (nullptr == list) {
+ // Convert to large list
+ header = GetLargeListPage<T>();
+ list = header->GetList<T>();
+ Y_VERIFY(header->Capacity >= oldSize);
+ header->Size = oldSize;
+ }
+
+ if (std::is_trivially_copyable<T>::value) {
+ memcpy(list, oldList, sizeof(T) * oldSize);
+ } else {
+ for (size_t i = 0; i < oldSize; ++i) {
+ ::WriteUnaligned<T>(list + i, ::ReadUnaligned<T>(oldList + i));
+ }
+ }
+ ReturnList(oldList);
+
+ if (nullptr == header) {
+ return list + oldSize;
+ } else if (header->Size < header->Capacity) {
+ return header->GetList<T>() + header->Size++;
+ }
+ } else {
+ header = GetLargeListHeader(list);
+ }
+
+ // Add new page to large list
+ TLargeListHeader* newPage = GetLargeListPage<T>();
+ ++newPage->Size;
+ list = newPage->GetList<T>();
+ header->Append(newPage);
+ return list;
+ }
+
+ template <typename T>
+ T* CloneList(const T* list) {
+ if (Y_LIKELY(LARGE_MARK != GetMark(list))) {
+ const size_t size = TListPoolBase::GetPartListSize(list);
+ T* clonedList = GetList<T>(size);
+ for (size_t i = 0; i < size; ++i) {
+ new (clonedList + i) T(list[i]);
+ }
+ return clonedList;
+ } else {
+ TLargeListHeader* lastListHeader = nullptr;
+ const TLargeListHeader* start = GetLargeListHeader(list)->Next();
+ const TLargeListHeader* header = start;
+ do {
+ TLargeListHeader* newHeader = GetLargeListPage<T>();
+ newHeader->Size = header->Size;
+ T* newList = newHeader->GetList<T>();
+ const T* list = header->GetList<T>();
+ for (size_t i = 0; i < header->Size; ++i) {
+ new (newList + i) T(list[i]);
+ }
+ if (lastListHeader) {
+ lastListHeader->Append(newHeader);
+ }
+ lastListHeader = newHeader;
+ header = header->Next();
+ } while (header != start);
+
+ return lastListHeader->GetList<T>();
+ }
+ }
+
+ template <typename T>
+ void ReturnList(T* list) {
+ static_assert(std::is_same<TPrimary, T>::value || std::is_same<TSecondary, T>::value, "Bad returned list type");
+ static constexpr size_t PoolNdx = static_cast<size_t>(!std::is_same<TPrimary, T>::value);
+ switch (GetMark(list)) {
+ case SMALL_MARK:
+ ReturnSmallList<PoolNdx, T>(GetListHeader(list), list);
+ break;
+ case MEDIUM_MARK:
+ ReturnMediumList<PoolNdx, T>(GetListHeader(list), list);
+ break;
+ case LARGE_MARK:
+ ReturnLargeList<T>(list);
+ break;
+ default:
+ Y_FAIL("Bad list address");
+ }
+ }
+
+ void Swap(TListPool& other) {
+ TListPoolBase::Swap(other);
+ DoSwap(Pools, other.Pools);
+ }
+
void PrintStat(IOutputStream& out) const {
- size_t usedPages = 0;
- if (std::is_same<TPrimary, TSecondary>::value) {
+ size_t usedPages = 0;
+ if (std::is_same<TPrimary, TSecondary>::value) {
usedPages = Pools[0].PrintStat(TStringBuf(""), out);
- } else {
+ } else {
usedPages = Pools[0].PrintStat(TStringBuf("Primary: "), out) + Pools[1].PrintStat(TStringBuf("Secondary: "), out);
- }
+ }
GetPagePool().PrintStat(usedPages, out);
- }
-
- TString DebugInfo() const {
- if (std::is_same<TPrimary, TSecondary>::value) {
- return Pools[0].DebugInfo();
- } else {
- return TString().append("Primary:\n").append(Pools[0].DebugInfo()).append("Secondary:\n").append(Pools[1].DebugInfo());
- }
- }
-
-private:
- template <typename T>
- static void DestroyRange(T* s, T* e) {
- if (!std::is_trivially_destructible<T>::value) {
- while (s != e) {
- --e;
- e->~T();
- }
- }
- }
-
- template <size_t PoolNdx, typename T>
- TListHeader* GetSmallListPage(size_t size) {
- Y_ASSERT(size > 1 && size <= MAX_SMALL_LIST_SIZE);
+ }
+
+ TString DebugInfo() const {
+ if (std::is_same<TPrimary, TSecondary>::value) {
+ return Pools[0].DebugInfo();
+ } else {
+ return TString().append("Primary:\n").append(Pools[0].DebugInfo()).append("Secondary:\n").append(Pools[1].DebugInfo());
+ }
+ }
+
+private:
+ template <typename T>
+ static void DestroyRange(T* s, T* e) {
+ if (!std::is_trivially_destructible<T>::value) {
+ while (s != e) {
+ --e;
+ e->~T();
+ }
+ }
+ }
+
+ template <size_t PoolNdx, typename T>
+ TListHeader* GetSmallListPage(size_t size) {
+ Y_ASSERT(size > 1 && size <= MAX_SMALL_LIST_SIZE);
TListType& pages = Pools[PoolNdx].SmallPages[size - 2];
- if (!pages.Empty()) {
- return pages.Front()->As<TListHeader>();
- }
- ui16 listCount = GetSmallPageCapacity<T>(size);
- Y_ASSERT(listCount >= 2);
+ if (!pages.Empty()) {
+ return pages.Front()->As<TListHeader>();
+ }
+ ui16 listCount = GetSmallPageCapacity<T>(size);
+ Y_ASSERT(listCount >= 2);
TListHeader* header = new (GetPagePool().GetPage()) TListHeader(SMALL_MARK, size, listCount);
- pages.PushFront(&header->ListItem);
- return header;
- }
-
- template <size_t PoolNdx, typename T>
- TListHeader* GetMediumListPage(size_t size) {
- Y_ASSERT(size > MAX_SMALL_LIST_SIZE && size <= TListPoolBase::GetMaxListSize<T>());
- size_t index = MostSignificantBit((size - 1) >> MostSignificantBitCT(MAX_SMALL_LIST_SIZE));
- Y_ASSERT(index < Pools[PoolNdx].MediumPages.size());
+ pages.PushFront(&header->ListItem);
+ return header;
+ }
+
+ template <size_t PoolNdx, typename T>
+ TListHeader* GetMediumListPage(size_t size) {
+ Y_ASSERT(size > MAX_SMALL_LIST_SIZE && size <= TListPoolBase::GetMaxListSize<T>());
+ size_t index = MostSignificantBit((size - 1) >> MostSignificantBitCT(MAX_SMALL_LIST_SIZE));
+ Y_ASSERT(index < Pools[PoolNdx].MediumPages.size());
TListType& pages = Pools[PoolNdx].MediumPages[index];
- if (!pages.Empty()) {
- return pages.Front()->As<TListHeader>();
- }
- ui16 listCapacity = FastClp2(size);
- ui16 listCount = GetMediumPageCapacity<T>(listCapacity);
- Y_ASSERT(listCount >= 2);
+ if (!pages.Empty()) {
+ return pages.Front()->As<TListHeader>();
+ }
+ ui16 listCapacity = FastClp2(size);
+ ui16 listCount = GetMediumPageCapacity<T>(listCapacity);
+ Y_ASSERT(listCount >= 2);
TListHeader* header = new (GetPagePool().GetPage()) TListHeader(MEDIUM_MARK, listCapacity, listCount);
- pages.PushFront(&header->ListItem);
- return header;
- }
-
- template <typename T>
- TLargeListHeader* GetLargeListPage() {
+ pages.PushFront(&header->ListItem);
+ return header;
+ }
+
+ template <typename T>
+ TLargeListHeader* GetLargeListPage() {
TLargeListHeader* const header = new (GetPagePool().GetPage()) TLargeListHeader(GetLargePageCapacity<T>());
- return header;
- }
-
- template <size_t PoolNdx, typename T>
- T* GetSmallList(TListHeader* listHeader) {
- if (0 == listHeader->FreeLists) {
- return nullptr;
- }
- const bool last = (0 == --listHeader->FreeLists);
- // Always keep at least ui16 per list in order to track collection of free lists
- const size_t byteListSize = AlignUp<size_t>(sizeof(T) * listHeader->ListSize, sizeof(ui16));
- ui16* l = reinterpret_cast<ui16*>(reinterpret_cast<ui8*>(listHeader + 1) + byteListSize * listHeader->FreeListOffset);
- // Distinguish first (0) and repeatedly (0x8000u) used lists
- if ((*l) & 0x8000u) {
- listHeader->FreeListOffset = ((*l) & 0x7FFF);
- } else {
- ++listHeader->FreeListOffset;
- if (!last) {
- // Mark next free list as first used
- *reinterpret_cast<ui16*>(reinterpret_cast<ui8*>(listHeader + 1) + byteListSize * listHeader->FreeListOffset) = 0u;
- }
- }
- if (last) {
- listHeader->ListItem.Unlink();
- Pools[PoolNdx].FullPages.PushBack(&listHeader->ListItem);
- }
- return reinterpret_cast<T*>(l);
- }
-
- template <size_t PoolNdx, typename T>
- T* GetMediumList(TListHeader* listHeader, size_t size) {
- if (0 == listHeader->FreeLists) {
- return nullptr;
- }
- const bool last = (0 == --listHeader->FreeLists);
- const size_t byteListSize = sizeof(T) * listHeader->ListSize + sizeof(ui16);
- ui16* l = reinterpret_cast<ui16*>(reinterpret_cast<ui8*>(listHeader + 1) + byteListSize * listHeader->FreeListOffset);
- // Distinguish first (0) and repeatedly (0x8000u) used lists
- if ((*l) & 0x8000u) {
- listHeader->FreeListOffset = ((*l) & 0x7FFF);
- } else {
- ++listHeader->FreeListOffset;
- if (!last) {
- // Mark next free list as first used
- *reinterpret_cast<ui16*>(reinterpret_cast<ui8*>(listHeader + 1) + byteListSize * listHeader->FreeListOffset) = 0u;
- }
- }
-
- if (last) {
- listHeader->ListItem.Unlink();
- Pools[PoolNdx].FullPages.PushBack(&listHeader->ListItem);
- }
- // For medium pages store the list size ahead
- *l = size;
- ++l;
- return reinterpret_cast<T*>(l);
- }
-
- template <size_t PoolNdx, typename T>
- void ReturnSmallList(TListHeader* listHeader, T* list) {
- DestroyRange(list, list + listHeader->ListSize);
- const size_t byteListSize = AlignUp<size_t>(sizeof(T) * listHeader->ListSize, sizeof(ui16));
- Y_ASSERT((reinterpret_cast<ui8*>(list) - reinterpret_cast<ui8*>(listHeader + 1)) % byteListSize == 0);
- const ui64 offset = (reinterpret_cast<ui8*>(list) - reinterpret_cast<ui8*>(listHeader + 1)) / byteListSize;
+ return header;
+ }
+
+ template <size_t PoolNdx, typename T>
+ T* GetSmallList(TListHeader* listHeader) {
+ if (0 == listHeader->FreeLists) {
+ return nullptr;
+ }
+ const bool last = (0 == --listHeader->FreeLists);
+ // Always keep at least ui16 per list in order to track collection of free lists
+ const size_t byteListSize = AlignUp<size_t>(sizeof(T) * listHeader->ListSize, sizeof(ui16));
+ ui16* l = reinterpret_cast<ui16*>(reinterpret_cast<ui8*>(listHeader + 1) + byteListSize * listHeader->FreeListOffset);
+ // Distinguish first (0) and repeatedly (0x8000u) used lists
+ if ((*l) & 0x8000u) {
+ listHeader->FreeListOffset = ((*l) & 0x7FFF);
+ } else {
+ ++listHeader->FreeListOffset;
+ if (!last) {
+ // Mark next free list as first used
+ *reinterpret_cast<ui16*>(reinterpret_cast<ui8*>(listHeader + 1) + byteListSize * listHeader->FreeListOffset) = 0u;
+ }
+ }
+ if (last) {
+ listHeader->ListItem.Unlink();
+ Pools[PoolNdx].FullPages.PushBack(&listHeader->ListItem);
+ }
+ return reinterpret_cast<T*>(l);
+ }
+
+ template <size_t PoolNdx, typename T>
+ T* GetMediumList(TListHeader* listHeader, size_t size) {
+ if (0 == listHeader->FreeLists) {
+ return nullptr;
+ }
+ const bool last = (0 == --listHeader->FreeLists);
+ const size_t byteListSize = sizeof(T) * listHeader->ListSize + sizeof(ui16);
+ ui16* l = reinterpret_cast<ui16*>(reinterpret_cast<ui8*>(listHeader + 1) + byteListSize * listHeader->FreeListOffset);
+ // Distinguish first (0) and repeatedly (0x8000u) used lists
+ if ((*l) & 0x8000u) {
+ listHeader->FreeListOffset = ((*l) & 0x7FFF);
+ } else {
+ ++listHeader->FreeListOffset;
+ if (!last) {
+ // Mark next free list as first used
+ *reinterpret_cast<ui16*>(reinterpret_cast<ui8*>(listHeader + 1) + byteListSize * listHeader->FreeListOffset) = 0u;
+ }
+ }
+
+ if (last) {
+ listHeader->ListItem.Unlink();
+ Pools[PoolNdx].FullPages.PushBack(&listHeader->ListItem);
+ }
+ // For medium pages store the list size ahead
+ *l = size;
+ ++l;
+ return reinterpret_cast<T*>(l);
+ }
+
+ template <size_t PoolNdx, typename T>
+ void ReturnSmallList(TListHeader* listHeader, T* list) {
+ DestroyRange(list, list + listHeader->ListSize);
+ const size_t byteListSize = AlignUp<size_t>(sizeof(T) * listHeader->ListSize, sizeof(ui16));
+ Y_ASSERT((reinterpret_cast<ui8*>(list) - reinterpret_cast<ui8*>(listHeader + 1)) % byteListSize == 0);
+ const ui64 offset = (reinterpret_cast<ui8*>(list) - reinterpret_cast<ui8*>(listHeader + 1)) / byteListSize;
Y_ASSERT(offset < TAlignedPagePool::POOL_PAGE_SIZE);
- *reinterpret_cast<ui16*>(list) = listHeader->FreeListOffset | 0x8000u;
- listHeader->FreeListOffset = offset;
- ++listHeader->FreeLists;
- if (1 == listHeader->FreeLists) {
- listHeader->ListItem.Unlink(); // Remove from full list
- Pools[PoolNdx].SmallPages[listHeader->ListSize - 2].PushFront(&listHeader->ListItem); // Add to partially used
- } else if (GetSmallPageCapacity<T>(listHeader->ListSize) == listHeader->FreeLists) {
- listHeader->ListItem.Unlink(); // Remove from partially used
- FreeListPage(listHeader);
- }
- }
-
- template <size_t PoolNdx, typename T>
- void ReturnMediumList(TListHeader* listHeader, T* list) {
- ui16* l = reinterpret_cast<ui16*>(list) - 1;
- DestroyRange(list, list + *l);
- Y_ASSERT((reinterpret_cast<ui8*>(l) - reinterpret_cast<ui8*>(listHeader + 1)) % (listHeader->ListSize * sizeof(T) + sizeof(ui16)) == 0);
- ui64 offset = (reinterpret_cast<ui8*>(l) - reinterpret_cast<ui8*>(listHeader + 1)) / (listHeader->ListSize * sizeof(T) + sizeof(ui16));
+ *reinterpret_cast<ui16*>(list) = listHeader->FreeListOffset | 0x8000u;
+ listHeader->FreeListOffset = offset;
+ ++listHeader->FreeLists;
+ if (1 == listHeader->FreeLists) {
+ listHeader->ListItem.Unlink(); // Remove from full list
+ Pools[PoolNdx].SmallPages[listHeader->ListSize - 2].PushFront(&listHeader->ListItem); // Add to partially used
+ } else if (GetSmallPageCapacity<T>(listHeader->ListSize) == listHeader->FreeLists) {
+ listHeader->ListItem.Unlink(); // Remove from partially used
+ FreeListPage(listHeader);
+ }
+ }
+
+ template <size_t PoolNdx, typename T>
+ void ReturnMediumList(TListHeader* listHeader, T* list) {
+ ui16* l = reinterpret_cast<ui16*>(list) - 1;
+ DestroyRange(list, list + *l);
+ Y_ASSERT((reinterpret_cast<ui8*>(l) - reinterpret_cast<ui8*>(listHeader + 1)) % (listHeader->ListSize * sizeof(T) + sizeof(ui16)) == 0);
+ ui64 offset = (reinterpret_cast<ui8*>(l) - reinterpret_cast<ui8*>(listHeader + 1)) / (listHeader->ListSize * sizeof(T) + sizeof(ui16));
Y_ASSERT(offset < TAlignedPagePool::POOL_PAGE_SIZE);
- *l = listHeader->FreeListOffset | 0x8000u;
- listHeader->FreeListOffset = offset;
- ++listHeader->FreeLists;
- if (1 == listHeader->FreeLists) {
- listHeader->ListItem.Unlink(); // Remove from full list
- const size_t index = MostSignificantBit((listHeader->ListSize - 1) >> MostSignificantBitCT(MAX_SMALL_LIST_SIZE));
- Y_ASSERT(index < Pools[PoolNdx].MediumPages.size());
- Pools[PoolNdx].MediumPages[index].PushFront(&listHeader->ListItem); // Add to partially used
- } else if (GetMediumPageCapacity<T>(listHeader->ListSize) == listHeader->FreeLists) {
- listHeader->ListItem.Unlink(); // Remove from partially used
- FreeListPage(listHeader);
- }
- }
-
- template <typename T>
- void ReturnLargeList(T* list) {
- TLargeListHeader* header = GetLargeListHeader(list);
- while (!header->ListItem.Empty()) {
- TLargeListHeader* next = header->Next();
- DestroyRange(next->GetList<T>(), next->GetList<T>() + next->Size);
- next->ListItem.Unlink();
- next->~TLargeListHeader();
+ *l = listHeader->FreeListOffset | 0x8000u;
+ listHeader->FreeListOffset = offset;
+ ++listHeader->FreeLists;
+ if (1 == listHeader->FreeLists) {
+ listHeader->ListItem.Unlink(); // Remove from full list
+ const size_t index = MostSignificantBit((listHeader->ListSize - 1) >> MostSignificantBitCT(MAX_SMALL_LIST_SIZE));
+ Y_ASSERT(index < Pools[PoolNdx].MediumPages.size());
+ Pools[PoolNdx].MediumPages[index].PushFront(&listHeader->ListItem); // Add to partially used
+ } else if (GetMediumPageCapacity<T>(listHeader->ListSize) == listHeader->FreeLists) {
+ listHeader->ListItem.Unlink(); // Remove from partially used
+ FreeListPage(listHeader);
+ }
+ }
+
+ template <typename T>
+ void ReturnLargeList(T* list) {
+ TLargeListHeader* header = GetLargeListHeader(list);
+ while (!header->ListItem.Empty()) {
+ TLargeListHeader* next = header->Next();
+ DestroyRange(next->GetList<T>(), next->GetList<T>() + next->Size);
+ next->ListItem.Unlink();
+ next->~TLargeListHeader();
GetPagePool().ReturnPage(next);
- }
- DestroyRange(header->GetList<T>(), header->GetList<T>() + header->Size);
- header->~TLargeListHeader();
+ }
+ DestroyRange(header->GetList<T>(), header->GetList<T>() + header->Size);
+ header->~TLargeListHeader();
GetPagePool().ReturnPage(header);
- }
-protected:
- std::array<TUsedPages, PoolCount> Pools;
-};
-
-#pragma pack(push, 1)
-template <typename T>
-struct TNode {
- enum {
- FlagEmpty = 0,
- FlagSingle = 1,
- FlagList = 2,
- };
-
- ui8 Flag;
- union {
- ui8 D1;
- ui8 D2[Max<size_t>(sizeof(T), sizeof(T*))];
- } Storage;
-
- TNode()
- : Flag(FlagEmpty)
- {
- }
- TNode(const TNode& n)
- : Flag(n.Flag)
- {
- MemCopy(Storage.D2, n.Storage.D2, sizeof(Storage.D2));
- }
- TNode(TNode&& n)
- : Flag(n.Flag)
- {
- MemCopy(Storage.D2, n.Storage.D2, sizeof(Storage.D2));
- n.Flag = FlagEmpty;
- }
-
- TNode& operator=(const TNode& n) {
- Flag = n.Flag;
- MemCopy(Storage.D2, n.Storage.D2, sizeof(Storage.D2));
- return *this;
- }
-
- TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader> Iter() {
- if (FlagSingle == Flag) {
- return TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader>(reinterpret_cast<T*>(&Storage), reinterpret_cast<T*>(&Storage) + 1);
- } else if (FlagList == Flag) {
- return TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader>(*reinterpret_cast<T**>(&Storage));
- }
- return TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader>();
- }
-
- TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader> Iter() const {
- if (FlagSingle == Flag) {
- return TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader>(reinterpret_cast<const T*>(&Storage), reinterpret_cast<const T*>(&Storage) + 1);
- } else if (FlagList == Flag) {
- return TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader>(*reinterpret_cast<const T* const*>(&Storage));
- }
- return TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader>();
- }
-
- void Set(T val) {
- Flag = FlagSingle;
- ::WriteUnaligned<T>(&Storage, val);
- }
-
- void SetList(T* list) {
- Flag = FlagList;
- ::WriteUnaligned<T*>(&Storage, list);
- }
-
- T Get() const {
- Y_VERIFY(FlagSingle == Flag);
- return ::ReadUnaligned<T>(&Storage);
- }
-
- T* GetList() {
- Y_VERIFY(FlagList == Flag);
- return *reinterpret_cast<T**>(&Storage);
- }
-
- const T* GetList() const {
- Y_VERIFY(FlagList == Flag);
- return *reinterpret_cast<const T* const*>(&Storage);
- }
-
- size_t Size() const {
- if (FlagEmpty == Flag) {
- return 0;
- } else if (FlagSingle == Flag) {
- return 1;
- } else {
- return TListPoolBase::GetFullListSize(GetList());
- }
- }
-
- void Clear() {
- if (FlagSingle == Flag) {
- reinterpret_cast<T*>(&Storage)->~T();
- }
- Flag = FlagEmpty;
- }
-};
-
-template <typename TKey, typename TValue>
-struct TKeyValuePair {
- using first_type = TKey;
- using second_type = TValue;
-
- TKeyValuePair() = default;
- TKeyValuePair(const TKeyValuePair&) = default;
- TKeyValuePair(TKeyValuePair&&) = default;
-
- TKeyValuePair(const std::pair<TKey, TValue>& p)
- : first(p.first)
- , second(p.second)
- {
- }
- TKeyValuePair(const TKey& k, const TValue& v)
- : first(k)
- , second(v)
- {
- }
- TKeyValuePair(TKey&& k, const TValue& v)
- : first(std::move(k))
- , second(v)
- {
- }
- TKeyValuePair(const TKey& k, TValue&& v)
- : first(k)
- , second(std::move(v))
- {
- }
- TKeyValuePair(TKey&& k, TValue&& v)
- : first(std::move(k))
- , second(std::move(v))
- {
- }
-
- TKeyValuePair& operator =(const TKeyValuePair&) = default;
- TKeyValuePair& operator =(TKeyValuePair&&) = default;
-
- TKey first;
- TValue second;
-};
-
-template <typename TKey, typename TValue>
-using TKeyNodePair = TKeyValuePair<TKey, TNode<TValue>>;
-
-#pragma pack(pop)
-
-template <typename TItemType,
- typename TKeyType,
- typename TKeyExtractor,
- typename TKeyHash,
- typename TKeyEqual,
- typename TSubItemType = TItemType
- >
-class TCompactHashBase {
-protected:
- using TItemNode = TNode<TItemType>;
- static_assert(sizeof(TItemNode) == 1 + Max<size_t>(sizeof(TItemType), sizeof(void*)), "Unexpected size");
-
-public:
- template <typename T>
- class TIteratorImpl {
- friend class TCompactHashBase;
- using TBucketIter = TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader>;
-
- // Full scan iterator
- TIteratorImpl(const TCompactHashBase* hash)
- : Hash(hash)
- , Bucket(0)
+ }
+protected:
+ std::array<TUsedPages, PoolCount> Pools;
+};
+
+#pragma pack(push, 1)
+template <typename T>
+struct TNode {
+ enum {
+ FlagEmpty = 0,
+ FlagSingle = 1,
+ FlagList = 2,
+ };
+
+ ui8 Flag;
+ union {
+ ui8 D1;
+ ui8 D2[Max<size_t>(sizeof(T), sizeof(T*))];
+ } Storage;
+
+ TNode()
+ : Flag(FlagEmpty)
+ {
+ }
+ TNode(const TNode& n)
+ : Flag(n.Flag)
+ {
+ MemCopy(Storage.D2, n.Storage.D2, sizeof(Storage.D2));
+ }
+ TNode(TNode&& n)
+ : Flag(n.Flag)
+ {
+ MemCopy(Storage.D2, n.Storage.D2, sizeof(Storage.D2));
+ n.Flag = FlagEmpty;
+ }
+
+ TNode& operator=(const TNode& n) {
+ Flag = n.Flag;
+ MemCopy(Storage.D2, n.Storage.D2, sizeof(Storage.D2));
+ return *this;
+ }
+
+ TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader> Iter() {
+ if (FlagSingle == Flag) {
+ return TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader>(reinterpret_cast<T*>(&Storage), reinterpret_cast<T*>(&Storage) + 1);
+ } else if (FlagList == Flag) {
+ return TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader>(*reinterpret_cast<T**>(&Storage));
+ }
+ return TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader>();
+ }
+
+ TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader> Iter() const {
+ if (FlagSingle == Flag) {
+ return TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader>(reinterpret_cast<const T*>(&Storage), reinterpret_cast<const T*>(&Storage) + 1);
+ } else if (FlagList == Flag) {
+ return TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader>(*reinterpret_cast<const T* const*>(&Storage));
+ }
+ return TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader>();
+ }
+
+ void Set(T val) {
+ Flag = FlagSingle;
+ ::WriteUnaligned<T>(&Storage, val);
+ }
+
+ void SetList(T* list) {
+ Flag = FlagList;
+ ::WriteUnaligned<T*>(&Storage, list);
+ }
+
+ T Get() const {
+ Y_VERIFY(FlagSingle == Flag);
+ return ::ReadUnaligned<T>(&Storage);
+ }
+
+ T* GetList() {
+ Y_VERIFY(FlagList == Flag);
+ return *reinterpret_cast<T**>(&Storage);
+ }
+
+ const T* GetList() const {
+ Y_VERIFY(FlagList == Flag);
+ return *reinterpret_cast<const T* const*>(&Storage);
+ }
+
+ size_t Size() const {
+ if (FlagEmpty == Flag) {
+ return 0;
+ } else if (FlagSingle == Flag) {
+ return 1;
+ } else {
+ return TListPoolBase::GetFullListSize(GetList());
+ }
+ }
+
+ void Clear() {
+ if (FlagSingle == Flag) {
+ reinterpret_cast<T*>(&Storage)->~T();
+ }
+ Flag = FlagEmpty;
+ }
+};
+
+template <typename TKey, typename TValue>
+struct TKeyValuePair {
+ using first_type = TKey;
+ using second_type = TValue;
+
+ TKeyValuePair() = default;
+ TKeyValuePair(const TKeyValuePair&) = default;
+ TKeyValuePair(TKeyValuePair&&) = default;
+
+ TKeyValuePair(const std::pair<TKey, TValue>& p)
+ : first(p.first)
+ , second(p.second)
+ {
+ }
+ TKeyValuePair(const TKey& k, const TValue& v)
+ : first(k)
+ , second(v)
+ {
+ }
+ TKeyValuePair(TKey&& k, const TValue& v)
+ : first(std::move(k))
+ , second(v)
+ {
+ }
+ TKeyValuePair(const TKey& k, TValue&& v)
+ : first(k)
+ , second(std::move(v))
+ {
+ }
+ TKeyValuePair(TKey&& k, TValue&& v)
+ : first(std::move(k))
+ , second(std::move(v))
+ {
+ }
+
+ TKeyValuePair& operator =(const TKeyValuePair&) = default;
+ TKeyValuePair& operator =(TKeyValuePair&&) = default;
+
+ TKey first;
+ TValue second;
+};
+
+template <typename TKey, typename TValue>
+using TKeyNodePair = TKeyValuePair<TKey, TNode<TValue>>;
+
+#pragma pack(pop)
+
+template <typename TItemType,
+ typename TKeyType,
+ typename TKeyExtractor,
+ typename TKeyHash,
+ typename TKeyEqual,
+ typename TSubItemType = TItemType
+ >
+class TCompactHashBase {
+protected:
+ using TItemNode = TNode<TItemType>;
+ static_assert(sizeof(TItemNode) == 1 + Max<size_t>(sizeof(TItemType), sizeof(void*)), "Unexpected size");
+
+public:
+ template <typename T>
+ class TIteratorImpl {
+ friend class TCompactHashBase;
+ using TBucketIter = TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader>;
+
+ // Full scan iterator
+ TIteratorImpl(const TCompactHashBase* hash)
+ : Hash(hash)
+ , Bucket(0)
, EndBucket(Hash->BucketsCount_)
- , Pos()
- {
- for (; Bucket < EndBucket; ++Bucket) {
- if (!Hash->IsEmptyBucket(Bucket)) {
- Pos = Hash->GetBucketIter(Bucket);
- break;
- }
- }
- }
-
- // Key iterator
- TIteratorImpl(const TCompactHashBase* hash, size_t bucket, const TBucketIter& pos)
- : Hash(hash)
- , Bucket(bucket)
- , EndBucket(bucket + 1)
- , Pos(pos)
- {
- }
-
- // Empty iterator
- TIteratorImpl() {
- }
-
- public:
- TIteratorImpl& operator=(const TIteratorImpl& rhs) {
- Hash = rhs.Hash;
- Bucket = rhs.Bucket;
- EndBucket = rhs.EndBucket;
- Pos = rhs.Pos;
- return *this;
- }
-
- bool Ok() const {
- return Bucket < EndBucket && Pos.Ok();
- }
-
- TIteratorImpl& operator++() {
- if (Bucket < EndBucket) {
- if ((++Pos).Ok()) {
- return *this;
- }
-
- for (++Bucket; Bucket < EndBucket; ++Bucket) {
- if (!Hash->IsEmptyBucket(Bucket)) {
- Pos = Hash->GetBucketIter(Bucket);
- break;
- }
- }
- }
- return *this;
- }
-
- T operator*() const {
- Y_ASSERT(Ok());
- return Pos.Get();
- }
-
- T Get() const {
- Y_ASSERT(Ok());
- return Pos.Get();
- }
-
- TIteratorImpl MakeCurrentKeyIter() const {
- Y_ASSERT(Ok());
- return TIteratorImpl(Hash, Bucket, TBucketIter(Pos.GetRaw(), Pos.GetRaw() + sizeof(T)));
- }
-
- private:
- const TCompactHashBase* Hash = nullptr;
- size_t Bucket = 0;
- size_t EndBucket = 0;
- TBucketIter Pos;
- };
-
- template <typename T>
- class TIteratorImpl<TKeyNodePair<TKeyType, T>> {
- friend class TCompactHashBase;
- using TBucketIter = TListPoolBase::TListIterator<const TKeyNodePair<TKeyType, T>, const TListPoolBase::TLargeListHeader>;
- using TValueIter = TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader>;
-
- // Full scan iterator
- TIteratorImpl(const TCompactHashBase* hash)
- : Hash(hash)
- , Bucket(0)
+ , Pos()
+ {
+ for (; Bucket < EndBucket; ++Bucket) {
+ if (!Hash->IsEmptyBucket(Bucket)) {
+ Pos = Hash->GetBucketIter(Bucket);
+ break;
+ }
+ }
+ }
+
+ // Key iterator
+ TIteratorImpl(const TCompactHashBase* hash, size_t bucket, const TBucketIter& pos)
+ : Hash(hash)
+ , Bucket(bucket)
+ , EndBucket(bucket + 1)
+ , Pos(pos)
+ {
+ }
+
+ // Empty iterator
+ TIteratorImpl() {
+ }
+
+ public:
+ TIteratorImpl& operator=(const TIteratorImpl& rhs) {
+ Hash = rhs.Hash;
+ Bucket = rhs.Bucket;
+ EndBucket = rhs.EndBucket;
+ Pos = rhs.Pos;
+ return *this;
+ }
+
+ bool Ok() const {
+ return Bucket < EndBucket && Pos.Ok();
+ }
+
+ TIteratorImpl& operator++() {
+ if (Bucket < EndBucket) {
+ if ((++Pos).Ok()) {
+ return *this;
+ }
+
+ for (++Bucket; Bucket < EndBucket; ++Bucket) {
+ if (!Hash->IsEmptyBucket(Bucket)) {
+ Pos = Hash->GetBucketIter(Bucket);
+ break;
+ }
+ }
+ }
+ return *this;
+ }
+
+ T operator*() const {
+ Y_ASSERT(Ok());
+ return Pos.Get();
+ }
+
+ T Get() const {
+ Y_ASSERT(Ok());
+ return Pos.Get();
+ }
+
+ TIteratorImpl MakeCurrentKeyIter() const {
+ Y_ASSERT(Ok());
+ return TIteratorImpl(Hash, Bucket, TBucketIter(Pos.GetRaw(), Pos.GetRaw() + sizeof(T)));
+ }
+
+ private:
+ const TCompactHashBase* Hash = nullptr;
+ size_t Bucket = 0;
+ size_t EndBucket = 0;
+ TBucketIter Pos;
+ };
+
+ template <typename T>
+ class TIteratorImpl<TKeyNodePair<TKeyType, T>> {
+ friend class TCompactHashBase;
+ using TBucketIter = TListPoolBase::TListIterator<const TKeyNodePair<TKeyType, T>, const TListPoolBase::TLargeListHeader>;
+ using TValueIter = TListPoolBase::TListIterator<const T, const TListPoolBase::TLargeListHeader>;
+
+ // Full scan iterator
+ TIteratorImpl(const TCompactHashBase* hash)
+ : Hash(hash)
+ , Bucket(0)
, EndBucket(Hash->BucketsCount_)
- {
- for (; Bucket < EndBucket; ++Bucket) {
- if (!Hash->IsEmptyBucket(Bucket)) {
- Pos = Hash->GetBucketIter(Bucket);
- SubPos = static_cast<const TKeyNodePair<TKeyType, T>*>(Pos.GetRaw())->second.Iter();
- break;
- }
- }
- }
-
- // Key iterator
- TIteratorImpl(const TCompactHashBase* hash, size_t bucket, const TBucketIter& pos)
- : Hash(hash)
- , Bucket(bucket)
- , EndBucket(bucket + 1)
- , Pos(pos)
- , SubPos(static_cast<const TKeyNodePair<TKeyType, T>*>(Pos.GetRaw())->second.Iter())
- {
- }
-
- // Empty iterator
- TIteratorImpl() {
- }
-
- public:
- TIteratorImpl& operator=(const TIteratorImpl& rhs) {
- Hash = rhs.Hash;
- Bucket = rhs.Bucket;
- EndBucket = rhs.EndBucket;
- Pos = rhs.Pos;
- SubPos = rhs.SubPos;
- return *this;
- }
-
- bool Ok() const {
- return Bucket < EndBucket;
- }
-
- TIteratorImpl& operator++() {
- return Shift(false);
- }
-
- TIteratorImpl& NextKey() {
- return Shift(true);
- }
-
- TKeyType GetKey() const {
- Y_ASSERT(Ok());
- return Pos.Get().first;
- }
-
- T GetValue() const {
- Y_ASSERT(Ok());
- return SubPos.Get();
- }
-
- T operator*() const {
- return GetValue();
- }
-
- TIteratorImpl MakeCurrentKeyIter() const {
- Y_ASSERT(Ok());
- return TIteratorImpl(Hash, Bucket, TBucketIter(Pos.GetRaw(), static_cast<const ui8*>(Pos.GetRaw()) + sizeof(TKeyNodePair<TKeyType, T>)));
- }
-
- private:
- TIteratorImpl& Shift(bool nextKey) {
- Y_ASSERT(Bucket < EndBucket);
- Y_ASSERT(SubPos.Ok());
- if (!nextKey && (++SubPos).Ok()) {
- return *this;
- }
-
- Y_ASSERT(Pos.Ok());
- if ((++Pos).Ok()) {
- SubPos = static_cast<const TKeyNodePair<TKeyType, T>*>(Pos.GetRaw())->second.Iter();
- return *this;
- } else {
- SubPos = TValueIter();
- }
-
- for (++Bucket; Bucket < EndBucket; ++Bucket) {
- if (!Hash->IsEmptyBucket(Bucket)) {
- Pos = Hash->GetBucketIter(Bucket);
- SubPos = static_cast<const TKeyNodePair<TKeyType, T>*>(Pos.GetRaw())->second.Iter();
- break;
- }
- }
-
- return *this;
- }
-
- private:
- const TCompactHashBase* Hash = nullptr;
- size_t Bucket = 0;
- size_t EndBucket = 0;
- TBucketIter Pos;
- TValueIter SubPos;
- };
-
-public:
- using TIterator = TIteratorImpl<TItemType>;
- using TBucketIterator = TListPoolBase::TListIterator<TItemType, TListPoolBase::TLargeListHeader>;
- using TConstBucketIterator = TListPoolBase::TListIterator<const TItemType, const TListPoolBase::TLargeListHeader>;
-
+ {
+ for (; Bucket < EndBucket; ++Bucket) {
+ if (!Hash->IsEmptyBucket(Bucket)) {
+ Pos = Hash->GetBucketIter(Bucket);
+ SubPos = static_cast<const TKeyNodePair<TKeyType, T>*>(Pos.GetRaw())->second.Iter();
+ break;
+ }
+ }
+ }
+
+ // Key iterator
+ TIteratorImpl(const TCompactHashBase* hash, size_t bucket, const TBucketIter& pos)
+ : Hash(hash)
+ , Bucket(bucket)
+ , EndBucket(bucket + 1)
+ , Pos(pos)
+ , SubPos(static_cast<const TKeyNodePair<TKeyType, T>*>(Pos.GetRaw())->second.Iter())
+ {
+ }
+
+ // Empty iterator
+ TIteratorImpl() {
+ }
+
+ public:
+ TIteratorImpl& operator=(const TIteratorImpl& rhs) {
+ Hash = rhs.Hash;
+ Bucket = rhs.Bucket;
+ EndBucket = rhs.EndBucket;
+ Pos = rhs.Pos;
+ SubPos = rhs.SubPos;
+ return *this;
+ }
+
+ bool Ok() const {
+ return Bucket < EndBucket;
+ }
+
+ TIteratorImpl& operator++() {
+ return Shift(false);
+ }
+
+ TIteratorImpl& NextKey() {
+ return Shift(true);
+ }
+
+ TKeyType GetKey() const {
+ Y_ASSERT(Ok());
+ return Pos.Get().first;
+ }
+
+ T GetValue() const {
+ Y_ASSERT(Ok());
+ return SubPos.Get();
+ }
+
+ T operator*() const {
+ return GetValue();
+ }
+
+ TIteratorImpl MakeCurrentKeyIter() const {
+ Y_ASSERT(Ok());
+ return TIteratorImpl(Hash, Bucket, TBucketIter(Pos.GetRaw(), static_cast<const ui8*>(Pos.GetRaw()) + sizeof(TKeyNodePair<TKeyType, T>)));
+ }
+
+ private:
+ TIteratorImpl& Shift(bool nextKey) {
+ Y_ASSERT(Bucket < EndBucket);
+ Y_ASSERT(SubPos.Ok());
+ if (!nextKey && (++SubPos).Ok()) {
+ return *this;
+ }
+
+ Y_ASSERT(Pos.Ok());
+ if ((++Pos).Ok()) {
+ SubPos = static_cast<const TKeyNodePair<TKeyType, T>*>(Pos.GetRaw())->second.Iter();
+ return *this;
+ } else {
+ SubPos = TValueIter();
+ }
+
+ for (++Bucket; Bucket < EndBucket; ++Bucket) {
+ if (!Hash->IsEmptyBucket(Bucket)) {
+ Pos = Hash->GetBucketIter(Bucket);
+ SubPos = static_cast<const TKeyNodePair<TKeyType, T>*>(Pos.GetRaw())->second.Iter();
+ break;
+ }
+ }
+
+ return *this;
+ }
+
+ private:
+ const TCompactHashBase* Hash = nullptr;
+ size_t Bucket = 0;
+ size_t EndBucket = 0;
+ TBucketIter Pos;
+ TValueIter SubPos;
+ };
+
+public:
+ using TIterator = TIteratorImpl<TItemType>;
+ using TBucketIterator = TListPoolBase::TListIterator<TItemType, TListPoolBase::TLargeListHeader>;
+ using TConstBucketIterator = TListPoolBase::TListIterator<const TItemType, const TListPoolBase::TLargeListHeader>;
+
TCompactHashBase(TAlignedPagePool& pagePool, size_t size = 0, const TKeyExtractor& keyExtractor = TKeyExtractor(),
- const TKeyHash& keyHash = TKeyHash(), const TKeyEqual& keyEqual = TKeyEqual())
+ const TKeyHash& keyHash = TKeyHash(), const TKeyEqual& keyEqual = TKeyEqual())
: ListPool_(pagePool)
, KeyExtractor_(keyExtractor)
- , KeyHash_(keyHash)
- , KeyEqual_(keyEqual)
- {
+ , KeyHash_(keyHash)
+ , KeyEqual_(keyEqual)
+ {
AllocateBuckets(FindNearestPrime(size));
- }
-
- TCompactHashBase(const TCompactHashBase& other)
- : Size_(other.Size_)
- , UniqSize_(other.UniqSize_)
- , MaxLoadFactor_(other.MaxLoadFactor_)
+ }
+
+ TCompactHashBase(const TCompactHashBase& other)
+ : Size_(other.Size_)
+ , UniqSize_(other.UniqSize_)
+ , MaxLoadFactor_(other.MaxLoadFactor_)
, ListPool_(other.ListPool_.GetPagePool())
- , KeyExtractor_(other.KeyExtractor_)
- , KeyHash_(other.KeyHash_)
- , KeyEqual_(other.KeyEqual_)
- {
+ , KeyExtractor_(other.KeyExtractor_)
+ , KeyHash_(other.KeyHash_)
+ , KeyEqual_(other.KeyEqual_)
+ {
AllocateBuckets(other.BucketsCount_);
for (size_t i = 0; i < other.BucketsCount_; ++i) {
auto& b = other.Buckets_[i];
- if (TItemNode::FlagSingle == b.Flag) {
- TItemType item = b.Get();
- UnconditionalInsert(KeyExtractor_(item)).Set(CloneItem(item));
- } else if (TItemNode::FlagList == b.Flag) {
- for (auto it = b.Iter(); it.Ok(); ++it) {
- UnconditionalInsert(KeyExtractor_(it.Get())).Set(CloneItem(it.Get()));
- }
- }
- }
- }
-
- TCompactHashBase(TCompactHashBase&& other)
- : Size_(std::move(other.Size_))
- , UniqSize_(std::move(other.UniqSize_))
- , MaxLoadFactor_(std::move(other.MaxLoadFactor_))
- , Buckets_(std::move(other.Buckets_))
+ if (TItemNode::FlagSingle == b.Flag) {
+ TItemType item = b.Get();
+ UnconditionalInsert(KeyExtractor_(item)).Set(CloneItem(item));
+ } else if (TItemNode::FlagList == b.Flag) {
+ for (auto it = b.Iter(); it.Ok(); ++it) {
+ UnconditionalInsert(KeyExtractor_(it.Get())).Set(CloneItem(it.Get()));
+ }
+ }
+ }
+ }
+
+ TCompactHashBase(TCompactHashBase&& other)
+ : Size_(std::move(other.Size_))
+ , UniqSize_(std::move(other.UniqSize_))
+ , MaxLoadFactor_(std::move(other.MaxLoadFactor_))
+ , Buckets_(std::move(other.Buckets_))
, BucketsCount_(std::move(other.BucketsCount_))
, BucketsMemory_(std::move(other.BucketsMemory_))
- , ListPool_(std::move(other.ListPool_))
- , KeyExtractor_(std::move(other.KeyExtractor_))
- , KeyHash_(std::move(other.KeyHash_))
- , KeyEqual_(std::move(other.KeyEqual_))
- {
+ , ListPool_(std::move(other.ListPool_))
+ , KeyExtractor_(std::move(other.KeyExtractor_))
+ , KeyHash_(std::move(other.KeyHash_))
+ , KeyEqual_(std::move(other.KeyEqual_))
+ {
other.Buckets_ = nullptr;
other.BucketsMemory_ = 0;
other.BucketsCount_ = 0;
- other.Size_ = 0;
- other.UniqSize_ = 0;
- other.MaxLoadFactor_ = 1.f;
- }
-
- ~TCompactHashBase() {
+ other.Size_ = 0;
+ other.UniqSize_ = 0;
+ other.MaxLoadFactor_ = 1.f;
+ }
+
+ ~TCompactHashBase() {
ClearImpl(true);
- }
-
- TCompactHashBase& operator= (const TCompactHashBase& other) {
- TCompactHashBase(other).Swap(*this);
- return *this;
- }
-
- TCompactHashBase& operator= (TCompactHashBase&& other) {
- TCompactHashBase(std::move(other)).Swap(*this);
- return *this;
- }
-
- void Clear() {
+ }
+
+ TCompactHashBase& operator= (const TCompactHashBase& other) {
+ TCompactHashBase(other).Swap(*this);
+ return *this;
+ }
+
+ TCompactHashBase& operator= (TCompactHashBase&& other) {
+ TCompactHashBase(std::move(other)).Swap(*this);
+ return *this;
+ }
+
+ void Clear() {
ClearImpl(false);
- }
-
- bool Has(const TKeyType& key) const {
- auto& b = Buckets_[GetBucket(key)];
- auto res = FindPos(b, key);
- return res.Ok();
- }
-
- bool Empty() const {
- return Size_ == 0;
- }
-
- size_t Size() const {
- return Size_;
- }
-
- size_t UniqSize() const {
- return UniqSize_;
- }
-
- const TKeyExtractor& GetKeyExtractor() const {
- return KeyExtractor_;
- }
-
- const TKeyHash& GetKeyHash() const {
- return KeyHash_;
- }
-
- const TKeyEqual& GetKeyEqual() const {
- return KeyEqual_;
- }
-
- float GetMaxLoadFactor() const {
- return MaxLoadFactor_;
- }
-
- void SetMaxLoadFactor(float factor) {
- Y_VERIFY(factor > 0);
- MaxLoadFactor_ = factor;
- }
-
- float GetLoadFactor() const {
+ }
+
+ bool Has(const TKeyType& key) const {
+ auto& b = Buckets_[GetBucket(key)];
+ auto res = FindPos(b, key);
+ return res.Ok();
+ }
+
+ bool Empty() const {
+ return Size_ == 0;
+ }
+
+ size_t Size() const {
+ return Size_;
+ }
+
+ size_t UniqSize() const {
+ return UniqSize_;
+ }
+
+ const TKeyExtractor& GetKeyExtractor() const {
+ return KeyExtractor_;
+ }
+
+ const TKeyHash& GetKeyHash() const {
+ return KeyHash_;
+ }
+
+ const TKeyEqual& GetKeyEqual() const {
+ return KeyEqual_;
+ }
+
+ float GetMaxLoadFactor() const {
+ return MaxLoadFactor_;
+ }
+
+ void SetMaxLoadFactor(float factor) {
+ Y_VERIFY(factor > 0);
+ MaxLoadFactor_ = factor;
+ }
+
+ float GetLoadFactor() const {
return float(UniqSize_) / float(BucketsCount_);
- }
-
- TAlignedPagePool& GetPagePool() const {
- return ListPool_.GetPagePool();
- }
-
- void Swap(TCompactHashBase& other) {
- DoSwap(Size_, other.Size_);
- DoSwap(UniqSize_, other.UniqSize_);
- DoSwap(MaxLoadFactor_, other.MaxLoadFactor_);
- DoSwap(Buckets_, other.Buckets_);
+ }
+
+ TAlignedPagePool& GetPagePool() const {
+ return ListPool_.GetPagePool();
+ }
+
+ void Swap(TCompactHashBase& other) {
+ DoSwap(Size_, other.Size_);
+ DoSwap(UniqSize_, other.UniqSize_);
+ DoSwap(MaxLoadFactor_, other.MaxLoadFactor_);
+ DoSwap(Buckets_, other.Buckets_);
DoSwap(BucketsCount_, other.BucketsCount_);
DoSwap(BucketsMemory_, other.BucketsMemory_);
- DoSwap(ListPool_, other.ListPool_);
- DoSwap(KeyExtractor_, other.KeyExtractor_);
- DoSwap(KeyHash_, other.KeyHash_);
- DoSwap(KeyEqual_, other.KeyEqual_);
- }
-
- // Full scan
- TIterator Iterate() const {
- return TIterator(this);
- }
-
- // Key scan
- TIterator Find(const TKeyType& key) const {
- size_t bucket = GetBucket(key);
- auto& b = Buckets_[bucket];
- auto pos = FindPos(b, key);
- if (pos.Ok()) {
- return TIterator(this, bucket, pos);
- }
- return TIterator();
- }
-
- size_t Count(const TKeyType& key) const {
- size_t bucket = GetBucket(key);
- auto& b = Buckets_[bucket];
- auto pos = FindPos(b, key);
- if (pos.Ok()) {
- return ValueCount(pos.Get());
- }
- return 0;
- }
-
- size_t GetBucketCount() const {
+ DoSwap(ListPool_, other.ListPool_);
+ DoSwap(KeyExtractor_, other.KeyExtractor_);
+ DoSwap(KeyHash_, other.KeyHash_);
+ DoSwap(KeyEqual_, other.KeyEqual_);
+ }
+
+ // Full scan
+ TIterator Iterate() const {
+ return TIterator(this);
+ }
+
+ // Key scan
+ TIterator Find(const TKeyType& key) const {
+ size_t bucket = GetBucket(key);
+ auto& b = Buckets_[bucket];
+ auto pos = FindPos(b, key);
+ if (pos.Ok()) {
+ return TIterator(this, bucket, pos);
+ }
+ return TIterator();
+ }
+
+ size_t Count(const TKeyType& key) const {
+ size_t bucket = GetBucket(key);
+ auto& b = Buckets_[bucket];
+ auto pos = FindPos(b, key);
+ if (pos.Ok()) {
+ return ValueCount(pos.Get());
+ }
+ return 0;
+ }
+
+ size_t GetBucketCount() const {
return BucketsCount_;
- }
-
- size_t GetBucket(const TKeyType& key) const {
+ }
+
+ size_t GetBucket(const TKeyType& key) const {
return NYql::VaryingHash(KeyHash_(key)) % BucketsCount_;
- }
-
- bool IsEmptyBucket(size_t bucket) const {
+ }
+
+ bool IsEmptyBucket(size_t bucket) const {
Y_ASSERT(bucket < BucketsCount_);
- return TItemNode::FlagEmpty == Buckets_[bucket].Flag;
- }
-
- size_t GetBucketSize(size_t bucket) const {
+ return TItemNode::FlagEmpty == Buckets_[bucket].Flag;
+ }
+
+ size_t GetBucketSize(size_t bucket) const {
Y_ASSERT(bucket < BucketsCount_);
- return Buckets_[bucket].Size();
- }
-
- TConstBucketIterator GetBucketIter(size_t bucket) const {
+ return Buckets_[bucket].Size();
+ }
+
+ TConstBucketIterator GetBucketIter(size_t bucket) const {
Y_ASSERT(bucket < BucketsCount_);
const TItemNode& b = Buckets_[bucket];
return b.Iter();
- }
-
- bool Rehash(size_t size) {
+ }
+
+ bool Rehash(size_t size) {
if (double(size) / BucketsCount_ <= MaxLoadFactor_) {
- return false;
- }
-
+ return false;
+ }
+
TItemNode* oldBuckets = Buckets_;
size_t oldBucketsCount = BucketsCount_;
size_t oldBucketsMemory = BucketsMemory_;
AllocateBuckets(FindNearestPrime(ceil(double(size) / MaxLoadFactor_)));
-
- Y_DEFER {
- for (size_t i = 0; i < oldBucketsCount; ++i) {
- auto& b = oldBuckets[i];
- if (TItemNode::FlagList == b.Flag) {
- ListPool_.ReturnList(b.GetList());
- }
- b.Clear();
- }
- FreeBuckets(oldBuckets, oldBucketsMemory);
- };
- try {
- for (size_t i = 0; i < oldBucketsCount; ++i) {
- auto& b = oldBuckets[i];
- if (TItemNode::FlagSingle == b.Flag) {
- TItemType item = b.Get();
- UnconditionalInsert(KeyExtractor_(item)).Set(item);
- } else if (TItemNode::FlagList == b.Flag) {
- for (TBucketIterator it = b.Iter(); it.Ok(); ++it) {
- UnconditionalInsert(KeyExtractor_(it.Get())).Set(it.Get());
- }
- }
- }
- } catch (...) {
- DoSwap(oldBuckets, Buckets_);
- DoSwap(oldBucketsCount, BucketsCount_);
- DoSwap(oldBucketsMemory, BucketsMemory_);
- throw;
- }
-
- return true;
- }
-
+
+ Y_DEFER {
+ for (size_t i = 0; i < oldBucketsCount; ++i) {
+ auto& b = oldBuckets[i];
+ if (TItemNode::FlagList == b.Flag) {
+ ListPool_.ReturnList(b.GetList());
+ }
+ b.Clear();
+ }
+ FreeBuckets(oldBuckets, oldBucketsMemory);
+ };
+ try {
+ for (size_t i = 0; i < oldBucketsCount; ++i) {
+ auto& b = oldBuckets[i];
+ if (TItemNode::FlagSingle == b.Flag) {
+ TItemType item = b.Get();
+ UnconditionalInsert(KeyExtractor_(item)).Set(item);
+ } else if (TItemNode::FlagList == b.Flag) {
+ for (TBucketIterator it = b.Iter(); it.Ok(); ++it) {
+ UnconditionalInsert(KeyExtractor_(it.Get())).Set(it.Get());
+ }
+ }
+ }
+ } catch (...) {
+ DoSwap(oldBuckets, Buckets_);
+ DoSwap(oldBucketsCount, BucketsCount_);
+ DoSwap(oldBucketsMemory, BucketsMemory_);
+ throw;
+ }
+
+ return true;
+ }
+
void PrintStat(IOutputStream& out) const {
- size_t empty = 0;
- size_t single = 0;
- size_t list = 0;
+ size_t empty = 0;
+ size_t single = 0;
+ size_t list = 0;
for (size_t i = 0; i < BucketsCount_; ++i) {
auto& b = Buckets_[i];
- if (TItemNode::FlagEmpty == b.Flag) {
- ++empty;
- } else if (TItemNode::FlagSingle == b.Flag) {
- ++single;
- } else {
- ++list;
- }
- }
- out << "Empty buckets: " << empty << Endl;
- out << "Single buckets: " << single << Endl;
- out << "List buckets: " << list << Endl;
- ListPool_.PrintStat(out);
- }
-
-protected:
+ if (TItemNode::FlagEmpty == b.Flag) {
+ ++empty;
+ } else if (TItemNode::FlagSingle == b.Flag) {
+ ++single;
+ } else {
+ ++list;
+ }
+ }
+ out << "Empty buckets: " << empty << Endl;
+ out << "Single buckets: " << single << Endl;
+ out << "List buckets: " << list << Endl;
+ ListPool_.PrintStat(out);
+ }
+
+protected:
void ClearImpl(bool fromDtor) {
for (size_t i = 0; i < BucketsCount_; ++i) {
ClearNode(Buckets_[i]);
@@ -1340,330 +1340,330 @@ protected:
GetPagePool().ReturnBlock(buckets, memory);
}
- TConstBucketIterator FindPos(const TNode<TItemType>& b, const TKeyType& key) const {
- if (TItemNode::FlagEmpty != b.Flag) {
- for (TConstBucketIterator it = b.Iter(); it.Ok(); ++it) {
- if (KeyEqual_(key, KeyExtractor_(it.Get()))) {
- return TConstBucketIterator(it.GetRaw(), static_cast<const TItemType*>(it.GetRaw()) + 1); // Limit iterator by current key only
- }
- }
- }
- return TConstBucketIterator();
- }
-
- TBucketIterator FindPos(TNode<TItemType>& b, const TKeyType& key) {
- if (TItemNode::FlagEmpty != b.Flag) {
- for (TBucketIterator it = b.Iter(); it.Ok(); ++it) {
- if (KeyEqual_(key, KeyExtractor_(it.Get()))) {
- return TBucketIterator(it.GetRaw(), static_cast<TItemType*>(it.GetRaw()) + 1); // Limit iterator by current key only
- }
- }
- }
- return TBucketIterator();
- }
-
- template <typename T>
- static size_t ValueCount(const TKeyNodePair<TKeyType, T>& item) {
- return item.second.Size();
- }
-
- template <typename T>
- static size_t ValueCount(const T& /*item*/) {
- return 1;
- }
-
- template <typename T>
- T CloneItem(const T& src) {
- return src;
- }
-
- template <typename T>
- TKeyNodePair<TKeyType, T> CloneItem(const TKeyNodePair<TKeyType, T>& src) {
- if (TNode<T>::FlagList == src.second.Flag) {
- TKeyNodePair<TKeyType, T> res(src.first, TNode<T>());
- res.second.SetList(ListPool_.template CloneList<T>(src.second.GetList()));
- return res;
- } else {
- return TKeyNodePair<TKeyType, T>(src.first, src.second);
- }
- }
-
- template <typename T>
- struct THasNodeValue : public std::false_type {};
-
- template <typename T>
- struct THasNodeValue<TKeyNodePair<TKeyType, T>> : public std::true_type {};
-
- template <typename T>
- void ClearNode(TNode<T>& b) {
- if (THasNodeValue<T>::value) {
- for (auto it = b.Iter(); it.Ok(); ++it) {
- ClearItem(*reinterpret_cast<T*>(it.GetRaw()));
- }
- }
-
- if (TNode<T>::FlagList == b.Flag) {
- ListPool_.ReturnList(b.GetList());
- }
- b.Clear();
- }
-
- template <typename T>
- void ClearItem(T& /*item*/) {
- }
-
- template <typename T>
- void ClearItem(TKeyNodePair<TKeyType, T>& item) {
- ClearNode(item.second);
- }
-
- std::pair<TBucketIterator, bool> InsertOrReplace(TKeyType key) {
- auto b = &Buckets_[GetBucket(key)];
- auto res = FindPos(*b, key);
- if (res.Ok()) {
- return {res, false};
- }
-
- if (Rehash(UniqSize_ + 1)) {
- // Update bucket after rehashing
- b = &Buckets_[GetBucket(key)];
- }
- TBucketIterator iter = ExpandBucket<TItemType>(*b);
- ++UniqSize_;
- ++Size_;
- return {iter, true};
- }
-
- TBucketIterator UnconditionalInsert(TKeyType key) {
- return ExpandBucket<TItemType>(Buckets_[GetBucket(key)]);
- }
-
- template <typename T>
- TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader> ExpandBucket(TNode<T>& b) {
- if (TNode<T>::FlagEmpty == b.Flag) {
- b.Flag = TNode<T>::FlagSingle;
- return b.Iter();
- } else if (TNode<T>::FlagSingle == b.Flag) {
- T* list = ListPool_.template GetList<T>(2);
- *list = b.Get();
- b.SetList(list);
- return TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader>(reinterpret_cast<ui8*>(list + 1), reinterpret_cast<ui8*>(list + 2));
- } else {
- T* list = b.GetList();
- T* pos = ListPool_.template IncrementList<T>(list);
- b.SetList(list);
- return TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader>(reinterpret_cast<ui8*>(pos), reinterpret_cast<ui8*>(pos + 1));
- }
- }
-
-protected:
- size_t Size_ = 0;
- size_t UniqSize_ = 0;
- float MaxLoadFactor_ = 1.f;
+ TConstBucketIterator FindPos(const TNode<TItemType>& b, const TKeyType& key) const {
+ if (TItemNode::FlagEmpty != b.Flag) {
+ for (TConstBucketIterator it = b.Iter(); it.Ok(); ++it) {
+ if (KeyEqual_(key, KeyExtractor_(it.Get()))) {
+ return TConstBucketIterator(it.GetRaw(), static_cast<const TItemType*>(it.GetRaw()) + 1); // Limit iterator by current key only
+ }
+ }
+ }
+ return TConstBucketIterator();
+ }
+
+ TBucketIterator FindPos(TNode<TItemType>& b, const TKeyType& key) {
+ if (TItemNode::FlagEmpty != b.Flag) {
+ for (TBucketIterator it = b.Iter(); it.Ok(); ++it) {
+ if (KeyEqual_(key, KeyExtractor_(it.Get()))) {
+ return TBucketIterator(it.GetRaw(), static_cast<TItemType*>(it.GetRaw()) + 1); // Limit iterator by current key only
+ }
+ }
+ }
+ return TBucketIterator();
+ }
+
+ template <typename T>
+ static size_t ValueCount(const TKeyNodePair<TKeyType, T>& item) {
+ return item.second.Size();
+ }
+
+ template <typename T>
+ static size_t ValueCount(const T& /*item*/) {
+ return 1;
+ }
+
+ template <typename T>
+ T CloneItem(const T& src) {
+ return src;
+ }
+
+ template <typename T>
+ TKeyNodePair<TKeyType, T> CloneItem(const TKeyNodePair<TKeyType, T>& src) {
+ if (TNode<T>::FlagList == src.second.Flag) {
+ TKeyNodePair<TKeyType, T> res(src.first, TNode<T>());
+ res.second.SetList(ListPool_.template CloneList<T>(src.second.GetList()));
+ return res;
+ } else {
+ return TKeyNodePair<TKeyType, T>(src.first, src.second);
+ }
+ }
+
+ template <typename T>
+ struct THasNodeValue : public std::false_type {};
+
+ template <typename T>
+ struct THasNodeValue<TKeyNodePair<TKeyType, T>> : public std::true_type {};
+
+ template <typename T>
+ void ClearNode(TNode<T>& b) {
+ if (THasNodeValue<T>::value) {
+ for (auto it = b.Iter(); it.Ok(); ++it) {
+ ClearItem(*reinterpret_cast<T*>(it.GetRaw()));
+ }
+ }
+
+ if (TNode<T>::FlagList == b.Flag) {
+ ListPool_.ReturnList(b.GetList());
+ }
+ b.Clear();
+ }
+
+ template <typename T>
+ void ClearItem(T& /*item*/) {
+ }
+
+ template <typename T>
+ void ClearItem(TKeyNodePair<TKeyType, T>& item) {
+ ClearNode(item.second);
+ }
+
+ std::pair<TBucketIterator, bool> InsertOrReplace(TKeyType key) {
+ auto b = &Buckets_[GetBucket(key)];
+ auto res = FindPos(*b, key);
+ if (res.Ok()) {
+ return {res, false};
+ }
+
+ if (Rehash(UniqSize_ + 1)) {
+ // Update bucket after rehashing
+ b = &Buckets_[GetBucket(key)];
+ }
+ TBucketIterator iter = ExpandBucket<TItemType>(*b);
+ ++UniqSize_;
+ ++Size_;
+ return {iter, true};
+ }
+
+ TBucketIterator UnconditionalInsert(TKeyType key) {
+ return ExpandBucket<TItemType>(Buckets_[GetBucket(key)]);
+ }
+
+ template <typename T>
+ TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader> ExpandBucket(TNode<T>& b) {
+ if (TNode<T>::FlagEmpty == b.Flag) {
+ b.Flag = TNode<T>::FlagSingle;
+ return b.Iter();
+ } else if (TNode<T>::FlagSingle == b.Flag) {
+ T* list = ListPool_.template GetList<T>(2);
+ *list = b.Get();
+ b.SetList(list);
+ return TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader>(reinterpret_cast<ui8*>(list + 1), reinterpret_cast<ui8*>(list + 2));
+ } else {
+ T* list = b.GetList();
+ T* pos = ListPool_.template IncrementList<T>(list);
+ b.SetList(list);
+ return TListPoolBase::TListIterator<T, TListPoolBase::TLargeListHeader>(reinterpret_cast<ui8*>(pos), reinterpret_cast<ui8*>(pos + 1));
+ }
+ }
+
+protected:
+ size_t Size_ = 0;
+ size_t UniqSize_ = 0;
+ float MaxLoadFactor_ = 1.f;
TItemNode* Buckets_ = nullptr;
size_t BucketsCount_ = 0;
size_t BucketsMemory_ = 0;
- TListPool<TItemType, TSubItemType> ListPool_;
-
- TKeyExtractor KeyExtractor_;
- TKeyHash KeyHash_;
- TKeyEqual KeyEqual_;
-};
-
-template <typename TKey,
- typename TValue,
- typename TKeyHash = THash<TKey>,
- typename TKeyEqual = TEqualTo<TKey>>
-class TCompactHash: public TCompactHashBase<TKeyValuePair<TKey, TValue>, TKey, TSelect1st, TKeyHash, TKeyEqual> {
-private:
- static_assert(std::is_trivially_destructible<TKey>::value
- && std::is_trivially_copy_assignable<TKey>::value
- && std::is_trivially_move_assignable<TKey>::value
- && std::is_trivially_copy_constructible<TKey>::value
- && std::is_trivially_move_constructible<TKey>::value
- , "Expected POD key type");
- static_assert(std::is_trivially_destructible<TValue>::value
- && std::is_trivially_copy_assignable<TValue>::value
- && std::is_trivially_move_assignable<TValue>::value
- && std::is_trivially_copy_constructible<TValue>::value
- && std::is_trivially_move_constructible<TValue>::value
- , "Expected POD value type");
-
- using TItem = TKeyValuePair<TKey, TValue>;
- using TBase = TCompactHashBase<TItem, TKey, TSelect1st, TKeyHash, TKeyEqual>;
-public:
+ TListPool<TItemType, TSubItemType> ListPool_;
+
+ TKeyExtractor KeyExtractor_;
+ TKeyHash KeyHash_;
+ TKeyEqual KeyEqual_;
+};
+
+template <typename TKey,
+ typename TValue,
+ typename TKeyHash = THash<TKey>,
+ typename TKeyEqual = TEqualTo<TKey>>
+class TCompactHash: public TCompactHashBase<TKeyValuePair<TKey, TValue>, TKey, TSelect1st, TKeyHash, TKeyEqual> {
+private:
+ static_assert(std::is_trivially_destructible<TKey>::value
+ && std::is_trivially_copy_assignable<TKey>::value
+ && std::is_trivially_move_assignable<TKey>::value
+ && std::is_trivially_copy_constructible<TKey>::value
+ && std::is_trivially_move_constructible<TKey>::value
+ , "Expected POD key type");
+ static_assert(std::is_trivially_destructible<TValue>::value
+ && std::is_trivially_copy_assignable<TValue>::value
+ && std::is_trivially_move_assignable<TValue>::value
+ && std::is_trivially_copy_constructible<TValue>::value
+ && std::is_trivially_move_constructible<TValue>::value
+ , "Expected POD value type");
+
+ using TItem = TKeyValuePair<TKey, TValue>;
+ using TBase = TCompactHashBase<TItem, TKey, TSelect1st, TKeyHash, TKeyEqual>;
+public:
TCompactHash(TAlignedPagePool& pagePool, size_t size = 0, const TKeyHash& keyHash = TKeyHash(), const TKeyEqual& keyEqual = TKeyEqual())
: TBase(pagePool, size, TSelect1st(), keyHash, keyEqual)
- {
- }
-
- TCompactHash(const TCompactHash& other)
- : TBase(other)
- {
- }
- TCompactHash(TCompactHash&& other)
- : TBase(std::move(other))
- {
- }
-
- TCompactHash& operator= (const TCompactHash& rhs) {
- TBase::operator =(rhs);
- return *this;
- }
-
- TCompactHash& operator= (TCompactHash&& rhs) {
- TBase::operator =(std::move(rhs));
- return *this;
- }
-
- bool Insert(TItem item) {
- auto res = TBase::InsertOrReplace(TBase::KeyExtractor_(item));
- res.first.Set(item);
- return res.second;
- }
-
- bool Insert(TKey key, TValue value) {
- auto res = TBase::InsertOrReplace(key);
- res.first.Set(TItem(key, value));
- return res.second;
- }
-
- bool InsertNew(TItem item) {
- auto res = TBase::InsertOrReplace(TBase::KeyExtractor_(item));
- if (res.second) {
- res.first.Set(item);
- }
- return res.second;
- }
-
- bool InsertNew(TKey key, TValue value) {
- auto res = TBase::InsertOrReplace(key);
- if (res.second) {
- res.first.Set(TItem(key, value));
- }
- return res.second;
- }
-};
-
-template <typename TKey,
- typename TValue,
- typename TKeyHash = THash<TKey>,
- typename TKeyEqual = TEqualTo<TKey>>
-class TCompactMultiHash: public TCompactHashBase<TKeyNodePair<TKey, TValue>, TKey, TSelect1st, TKeyHash, TKeyEqual, TValue> {
-private:
- static_assert(std::is_trivially_destructible<TKey>::value
- && std::is_trivially_copy_assignable<TKey>::value
- && std::is_trivially_move_assignable<TKey>::value
- && std::is_trivially_copy_constructible<TKey>::value
- && std::is_trivially_move_constructible<TKey>::value
- , "Expected POD key type");
- static_assert(std::is_trivially_destructible<TValue>::value
- && std::is_trivially_copy_assignable<TValue>::value
- && std::is_trivially_move_assignable<TValue>::value
- && std::is_trivially_copy_constructible<TValue>::value
- && std::is_trivially_move_constructible<TValue>::value
- , "Expected POD value type");
-
- using TUserItem = std::pair<TKey, TValue>;
- using TStoreItem = TKeyNodePair<TKey, TValue>;
- using TBase = TCompactHashBase<TStoreItem, TKey, TSelect1st, TKeyHash, TKeyEqual, TValue>;
-
- static_assert(sizeof(TStoreItem) == sizeof(TKey) + sizeof(TNode<TValue>), "Unexpected size");
-
-public:
+ {
+ }
+
+ TCompactHash(const TCompactHash& other)
+ : TBase(other)
+ {
+ }
+ TCompactHash(TCompactHash&& other)
+ : TBase(std::move(other))
+ {
+ }
+
+ TCompactHash& operator= (const TCompactHash& rhs) {
+ TBase::operator =(rhs);
+ return *this;
+ }
+
+ TCompactHash& operator= (TCompactHash&& rhs) {
+ TBase::operator =(std::move(rhs));
+ return *this;
+ }
+
+ bool Insert(TItem item) {
+ auto res = TBase::InsertOrReplace(TBase::KeyExtractor_(item));
+ res.first.Set(item);
+ return res.second;
+ }
+
+ bool Insert(TKey key, TValue value) {
+ auto res = TBase::InsertOrReplace(key);
+ res.first.Set(TItem(key, value));
+ return res.second;
+ }
+
+ bool InsertNew(TItem item) {
+ auto res = TBase::InsertOrReplace(TBase::KeyExtractor_(item));
+ if (res.second) {
+ res.first.Set(item);
+ }
+ return res.second;
+ }
+
+ bool InsertNew(TKey key, TValue value) {
+ auto res = TBase::InsertOrReplace(key);
+ if (res.second) {
+ res.first.Set(TItem(key, value));
+ }
+ return res.second;
+ }
+};
+
+template <typename TKey,
+ typename TValue,
+ typename TKeyHash = THash<TKey>,
+ typename TKeyEqual = TEqualTo<TKey>>
+class TCompactMultiHash: public TCompactHashBase<TKeyNodePair<TKey, TValue>, TKey, TSelect1st, TKeyHash, TKeyEqual, TValue> {
+private:
+ static_assert(std::is_trivially_destructible<TKey>::value
+ && std::is_trivially_copy_assignable<TKey>::value
+ && std::is_trivially_move_assignable<TKey>::value
+ && std::is_trivially_copy_constructible<TKey>::value
+ && std::is_trivially_move_constructible<TKey>::value
+ , "Expected POD key type");
+ static_assert(std::is_trivially_destructible<TValue>::value
+ && std::is_trivially_copy_assignable<TValue>::value
+ && std::is_trivially_move_assignable<TValue>::value
+ && std::is_trivially_copy_constructible<TValue>::value
+ && std::is_trivially_move_constructible<TValue>::value
+ , "Expected POD value type");
+
+ using TUserItem = std::pair<TKey, TValue>;
+ using TStoreItem = TKeyNodePair<TKey, TValue>;
+ using TBase = TCompactHashBase<TStoreItem, TKey, TSelect1st, TKeyHash, TKeyEqual, TValue>;
+
+ static_assert(sizeof(TStoreItem) == sizeof(TKey) + sizeof(TNode<TValue>), "Unexpected size");
+
+public:
TCompactMultiHash(TAlignedPagePool& pagePool, size_t size = 0, const TKeyHash& keyHash = TKeyHash(), const TKeyEqual& keyEqual = TKeyEqual())
: TBase(pagePool, size, TSelect1st(), keyHash, keyEqual)
- {
- }
- TCompactMultiHash(const TCompactMultiHash& other)
- : TBase(other)
- {
- }
- TCompactMultiHash(TCompactMultiHash&& other)
- : TBase(std::move(other))
- {
- }
-
- TCompactMultiHash& operator= (const TCompactMultiHash& rhs) {
- TBase::operator =(rhs);
- return *this;
- }
-
- TCompactMultiHash& operator= (TCompactMultiHash&& rhs) {
- TBase::operator =(std::move(rhs));
- return *this;
- }
-
- void Insert(const TUserItem& item) {
- GetOrInsert(item.first).Set(item.second);
- }
-
- void Insert(TKey key, TValue value) {
- GetOrInsert(key).Set(value);
- }
-
-protected:
- template <typename TKeyType>
- TListPoolBase::TListIterator<TValue, TListPoolBase::TLargeListHeader> GetOrInsert(TKeyType key) {
- auto res = TBase::InsertOrReplace(key);
- if (res.second) {
- res.first.Set(TStoreItem(key, TNode<TValue>()));
- }
- auto valueIter = TBase::template ExpandBucket<TValue>(reinterpret_cast<TStoreItem*>(res.first.GetRaw())->second);
- if (!res.second) {
- ++TBase::Size_;
- }
- return valueIter;
- }
-};
-
-template <typename TKey,
- typename TKeyHash = THash<TKey>,
- typename TKeyEqual = TEqualTo<TKey>>
-class TCompactHashSet: public TCompactHashBase<TKey, TKey, TIdentity, TKeyHash, TKeyEqual> {
-private:
- static_assert(std::is_trivially_destructible<TKey>::value
- && std::is_trivially_copy_assignable<TKey>::value
- && std::is_trivially_move_assignable<TKey>::value
- && std::is_trivially_copy_constructible<TKey>::value
- && std::is_trivially_move_constructible<TKey>::value
- , "Expected POD key type");
-
- using TBase = TCompactHashBase<TKey, TKey, TIdentity, TKeyHash, TKeyEqual>;
-
-public:
+ {
+ }
+ TCompactMultiHash(const TCompactMultiHash& other)
+ : TBase(other)
+ {
+ }
+ TCompactMultiHash(TCompactMultiHash&& other)
+ : TBase(std::move(other))
+ {
+ }
+
+ TCompactMultiHash& operator= (const TCompactMultiHash& rhs) {
+ TBase::operator =(rhs);
+ return *this;
+ }
+
+ TCompactMultiHash& operator= (TCompactMultiHash&& rhs) {
+ TBase::operator =(std::move(rhs));
+ return *this;
+ }
+
+ void Insert(const TUserItem& item) {
+ GetOrInsert(item.first).Set(item.second);
+ }
+
+ void Insert(TKey key, TValue value) {
+ GetOrInsert(key).Set(value);
+ }
+
+protected:
+ template <typename TKeyType>
+ TListPoolBase::TListIterator<TValue, TListPoolBase::TLargeListHeader> GetOrInsert(TKeyType key) {
+ auto res = TBase::InsertOrReplace(key);
+ if (res.second) {
+ res.first.Set(TStoreItem(key, TNode<TValue>()));
+ }
+ auto valueIter = TBase::template ExpandBucket<TValue>(reinterpret_cast<TStoreItem*>(res.first.GetRaw())->second);
+ if (!res.second) {
+ ++TBase::Size_;
+ }
+ return valueIter;
+ }
+};
+
+template <typename TKey,
+ typename TKeyHash = THash<TKey>,
+ typename TKeyEqual = TEqualTo<TKey>>
+class TCompactHashSet: public TCompactHashBase<TKey, TKey, TIdentity, TKeyHash, TKeyEqual> {
+private:
+ static_assert(std::is_trivially_destructible<TKey>::value
+ && std::is_trivially_copy_assignable<TKey>::value
+ && std::is_trivially_move_assignable<TKey>::value
+ && std::is_trivially_copy_constructible<TKey>::value
+ && std::is_trivially_move_constructible<TKey>::value
+ , "Expected POD key type");
+
+ using TBase = TCompactHashBase<TKey, TKey, TIdentity, TKeyHash, TKeyEqual>;
+
+public:
TCompactHashSet(TAlignedPagePool& pagePool, size_t size = 0, const TKeyHash& keyHash = TKeyHash(), const TKeyEqual& keyEqual = TKeyEqual())
: TBase(pagePool, size, TIdentity(), keyHash, keyEqual)
- {
- }
- TCompactHashSet(const TCompactHashSet& other)
- : TBase(other)
- {
- }
- TCompactHashSet(TCompactHashSet&& other)
- : TBase(std::move(other))
- {
- }
-
- TCompactHashSet& operator= (const TCompactHashSet& rhs) {
- TBase::operator =(rhs);
- return *this;
- }
-
- TCompactHashSet& operator= (TCompactHashSet&& rhs) {
- TBase::operator =(std::move(rhs));
- return *this;
- }
-
- bool Insert(TKey key) {
- auto res = TBase::InsertOrReplace(key);
- if (res.second) {
- res.first.Set(key);
- }
- return res.second;
- }
-};
-
-} // NCHash
-
-} // NKikimr
+ {
+ }
+ TCompactHashSet(const TCompactHashSet& other)
+ : TBase(other)
+ {
+ }
+ TCompactHashSet(TCompactHashSet&& other)
+ : TBase(std::move(other))
+ {
+ }
+
+ TCompactHashSet& operator= (const TCompactHashSet& rhs) {
+ TBase::operator =(rhs);
+ return *this;
+ }
+
+ TCompactHashSet& operator= (TCompactHashSet&& rhs) {
+ TBase::operator =(std::move(rhs));
+ return *this;
+ }
+
+ bool Insert(TKey key) {
+ auto res = TBase::InsertOrReplace(key);
+ if (res.second) {
+ res.first.Set(key);
+ }
+ return res.second;
+ }
+};
+
+} // NCHash
+
+} // NKikimr
diff --git a/ydb/library/yql/minikql/compact_hash_ut.cpp b/ydb/library/yql/minikql/compact_hash_ut.cpp
index 02278f64cb..128924ff79 100644
--- a/ydb/library/yql/minikql/compact_hash_ut.cpp
+++ b/ydb/library/yql/minikql/compact_hash_ut.cpp
@@ -1,404 +1,404 @@
-#include "compact_hash.h"
-
+#include "compact_hash.h"
+
#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/generic/bitmap.h>
-#include <util/generic/ylimits.h>
-#include <util/generic/hash_set.h>
-#include <util/generic/maybe.h>
-#include <util/generic/xrange.h>
-#include <util/random/shuffle.h>
-
-using namespace NKikimr;
-using namespace NKikimr::NCHash;
-
+
+#include <util/generic/bitmap.h>
+#include <util/generic/ylimits.h>
+#include <util/generic/hash_set.h>
+#include <util/generic/maybe.h>
+#include <util/generic/xrange.h>
+#include <util/random/shuffle.h>
+
+using namespace NKikimr;
+using namespace NKikimr::NCHash;
+
Y_UNIT_TEST_SUITE(TCompactHashTest) {
-
- template <typename TItem>
- void TestListPoolPagesImpl(size_t listSize, ui16 countOfLists, size_t expectedListCapacity, ui32 expectedMark) {
- using TPool = TListPool<TItem>;
- TAlignedPagePool pagePool;
+
+ template <typename TItem>
+ void TestListPoolPagesImpl(size_t listSize, ui16 countOfLists, size_t expectedListCapacity, ui32 expectedMark) {
+ using TPool = TListPool<TItem>;
+ TAlignedPagePool pagePool;
TPool pool(pagePool);
- UNIT_ASSERT(countOfLists > 1);
-
+ UNIT_ASSERT(countOfLists > 1);
+
THashSet<TItem*> lists;
-
- void* pageAddr = nullptr;
- for (size_t i = 0; i < countOfLists / 2; ++i) {
- TItem* l = pool.template GetList<TItem>(listSize);
- UNIT_ASSERT_VALUES_EQUAL(expectedMark, TPool::GetMark(l));
- UNIT_ASSERT_VALUES_EQUAL(listSize, TListPoolBase::GetPartListSize(l));
- UNIT_ASSERT_VALUES_EQUAL(expectedListCapacity, TListPoolBase::GetListCapacity(l));
- if (0 == i) {
- pageAddr = TAlignedPagePool::GetPageStart(l);
- } else {
- // All lists are from the same page
- UNIT_ASSERT_VALUES_EQUAL(pageAddr, TAlignedPagePool::GetPageStart(l));
- }
- UNIT_ASSERT(lists.insert(l).second);
- }
- // Return all lists except one
- while (lists.size() > 1) {
- auto it = lists.begin();
- pool.ReturnList(*it);
- lists.erase(it);
- }
-
- for (size_t i = 1; i < countOfLists; ++i) {
- TItem* l = pool.template GetList<TItem>(listSize);
- UNIT_ASSERT_VALUES_EQUAL(expectedMark, TPool::GetMark(l));
- UNIT_ASSERT_VALUES_EQUAL(listSize, TListPoolBase::GetPartListSize(l));
- UNIT_ASSERT_VALUES_EQUAL(expectedListCapacity, TListPoolBase::GetListCapacity(l));
- UNIT_ASSERT_VALUES_EQUAL(pageAddr, TAlignedPagePool::GetPageStart(l));
- UNIT_ASSERT(lists.insert(l).second);
- }
-
- for (auto l: lists) {
- pool.ReturnList(l);
- }
-
+
+ void* pageAddr = nullptr;
+ for (size_t i = 0; i < countOfLists / 2; ++i) {
+ TItem* l = pool.template GetList<TItem>(listSize);
+ UNIT_ASSERT_VALUES_EQUAL(expectedMark, TPool::GetMark(l));
+ UNIT_ASSERT_VALUES_EQUAL(listSize, TListPoolBase::GetPartListSize(l));
+ UNIT_ASSERT_VALUES_EQUAL(expectedListCapacity, TListPoolBase::GetListCapacity(l));
+ if (0 == i) {
+ pageAddr = TAlignedPagePool::GetPageStart(l);
+ } else {
+ // All lists are from the same page
+ UNIT_ASSERT_VALUES_EQUAL(pageAddr, TAlignedPagePool::GetPageStart(l));
+ }
+ UNIT_ASSERT(lists.insert(l).second);
+ }
+ // Return all lists except one
+ while (lists.size() > 1) {
+ auto it = lists.begin();
+ pool.ReturnList(*it);
+ lists.erase(it);
+ }
+
+ for (size_t i = 1; i < countOfLists; ++i) {
+ TItem* l = pool.template GetList<TItem>(listSize);
+ UNIT_ASSERT_VALUES_EQUAL(expectedMark, TPool::GetMark(l));
+ UNIT_ASSERT_VALUES_EQUAL(listSize, TListPoolBase::GetPartListSize(l));
+ UNIT_ASSERT_VALUES_EQUAL(expectedListCapacity, TListPoolBase::GetListCapacity(l));
+ UNIT_ASSERT_VALUES_EQUAL(pageAddr, TAlignedPagePool::GetPageStart(l));
+ UNIT_ASSERT(lists.insert(l).second);
+ }
+
+ for (auto l: lists) {
+ pool.ReturnList(l);
+ }
+
THashSet<TItem*> lists2;
- for (size_t i = 0; i < countOfLists; ++i) {
- TItem* l = pool.template GetList<TItem>(listSize);
- // All lists are from the same page
- UNIT_ASSERT_VALUES_EQUAL(pageAddr, TAlignedPagePool::GetPageStart(l));
- UNIT_ASSERT(1 == lists.erase(l));
- UNIT_ASSERT(lists2.insert(l).second);
- }
- TItem* l = pool.template GetList<TItem>(listSize); // New page
- UNIT_ASSERT_VALUES_UNEQUAL(pageAddr, TAlignedPagePool::GetPageStart(l));
- UNIT_ASSERT_VALUES_EQUAL(listSize, TListPoolBase::GetPartListSize(l));
- UNIT_ASSERT_VALUES_EQUAL(expectedListCapacity, TListPoolBase::GetListCapacity(l));
- pool.ReturnList(l);
- for (auto l: lists2) {
- pool.ReturnList(l);
- }
- }
-
- template <typename TItem>
- void TestListPoolLargeImpl() {
- using TPool = TListPool<TItem>;
- TAlignedPagePool pagePool;
+ for (size_t i = 0; i < countOfLists; ++i) {
+ TItem* l = pool.template GetList<TItem>(listSize);
+ // All lists are from the same page
+ UNIT_ASSERT_VALUES_EQUAL(pageAddr, TAlignedPagePool::GetPageStart(l));
+ UNIT_ASSERT(1 == lists.erase(l));
+ UNIT_ASSERT(lists2.insert(l).second);
+ }
+ TItem* l = pool.template GetList<TItem>(listSize); // New page
+ UNIT_ASSERT_VALUES_UNEQUAL(pageAddr, TAlignedPagePool::GetPageStart(l));
+ UNIT_ASSERT_VALUES_EQUAL(listSize, TListPoolBase::GetPartListSize(l));
+ UNIT_ASSERT_VALUES_EQUAL(expectedListCapacity, TListPoolBase::GetListCapacity(l));
+ pool.ReturnList(l);
+ for (auto l: lists2) {
+ pool.ReturnList(l);
+ }
+ }
+
+ template <typename TItem>
+ void TestListPoolLargeImpl() {
+ using TPool = TListPool<TItem>;
+ TAlignedPagePool pagePool;
TPool pool(pagePool);
- const size_t listSize = TListPoolBase::GetMaxListSize<TItem>();
- TItem* l = pool.template GetList<TItem>(listSize);
- pool.template IncrementList<TItem>(l);
- UNIT_ASSERT_VALUES_EQUAL((ui32)TListPoolBase::LARGE_MARK, TListPoolBase::GetMark(l));
- UNIT_ASSERT_VALUES_EQUAL(listSize + 1, TListPoolBase::GetPartListSize(l));
- TListPoolBase::SetPartListSize(l, TListPoolBase::GetListCapacity(l));
- pool.template IncrementList<TItem>(l);
- UNIT_ASSERT_VALUES_EQUAL(1, TListPoolBase::GetPartListSize(l));
- UNIT_ASSERT_VALUES_EQUAL(1 + TListPoolBase::GetListCapacity(l), TListPoolBase::GetFullListSize(l));
- pool.ReturnList(l);
- }
-
+ const size_t listSize = TListPoolBase::GetMaxListSize<TItem>();
+ TItem* l = pool.template GetList<TItem>(listSize);
+ pool.template IncrementList<TItem>(l);
+ UNIT_ASSERT_VALUES_EQUAL((ui32)TListPoolBase::LARGE_MARK, TListPoolBase::GetMark(l));
+ UNIT_ASSERT_VALUES_EQUAL(listSize + 1, TListPoolBase::GetPartListSize(l));
+ TListPoolBase::SetPartListSize(l, TListPoolBase::GetListCapacity(l));
+ pool.template IncrementList<TItem>(l);
+ UNIT_ASSERT_VALUES_EQUAL(1, TListPoolBase::GetPartListSize(l));
+ UNIT_ASSERT_VALUES_EQUAL(1 + TListPoolBase::GetListCapacity(l), TListPoolBase::GetFullListSize(l));
+ pool.ReturnList(l);
+ }
+
Y_UNIT_TEST(TestListPoolSmallPagesByte) {
- ui16 count = TListPoolBase::GetSmallPageCapacity<ui8>(2);
- TestListPoolPagesImpl<ui8>(2, count, 2, TListPoolBase::SMALL_MARK);
- }
-
+ ui16 count = TListPoolBase::GetSmallPageCapacity<ui8>(2);
+ TestListPoolPagesImpl<ui8>(2, count, 2, TListPoolBase::SMALL_MARK);
+ }
+
Y_UNIT_TEST(TestListPoolMediumPagesByte) {
- size_t listSize = TListPoolBase::MAX_SMALL_LIST_SIZE + 1;
- ui16 count = TListPoolBase::GetMediumPageCapacity<ui8>(listSize);
- TestListPoolPagesImpl<ui8>(listSize, count, FastClp2(listSize), TListPoolBase::MEDIUM_MARK);
- }
-
+ size_t listSize = TListPoolBase::MAX_SMALL_LIST_SIZE + 1;
+ ui16 count = TListPoolBase::GetMediumPageCapacity<ui8>(listSize);
+ TestListPoolPagesImpl<ui8>(listSize, count, FastClp2(listSize), TListPoolBase::MEDIUM_MARK);
+ }
+
Y_UNIT_TEST(TestListPoolLargPagesByte) {
- TestListPoolLargeImpl<ui8>();
- }
-
+ TestListPoolLargeImpl<ui8>();
+ }
+
Y_UNIT_TEST(TestListPoolSmallPagesUi64) {
- ui16 count = TListPoolBase::GetSmallPageCapacity<ui64>(2);
- TestListPoolPagesImpl<ui64>(2, count, 2, TListPoolBase::SMALL_MARK);
- }
-
+ ui16 count = TListPoolBase::GetSmallPageCapacity<ui64>(2);
+ TestListPoolPagesImpl<ui64>(2, count, 2, TListPoolBase::SMALL_MARK);
+ }
+
Y_UNIT_TEST(TestListPoolMediumPagesUi64) {
- size_t listSize = TListPoolBase::MAX_SMALL_LIST_SIZE + 1;
- ui16 count = TListPoolBase::GetMediumPageCapacity<ui64>(listSize);
- TestListPoolPagesImpl<ui64>(listSize, count, FastClp2(listSize), TListPoolBase::MEDIUM_MARK);
- }
-
+ size_t listSize = TListPoolBase::MAX_SMALL_LIST_SIZE + 1;
+ ui16 count = TListPoolBase::GetMediumPageCapacity<ui64>(listSize);
+ TestListPoolPagesImpl<ui64>(listSize, count, FastClp2(listSize), TListPoolBase::MEDIUM_MARK);
+ }
+
Y_UNIT_TEST(TestListPoolLargPagesUi64) {
- TestListPoolLargeImpl<ui64>();
- }
-
- struct TItem {
- ui8 A[256];
- };
-
+ TestListPoolLargeImpl<ui64>();
+ }
+
+ struct TItem {
+ ui8 A[256];
+ };
+
Y_UNIT_TEST(TestListPoolSmallPagesObj) {
- ui16 count = TListPoolBase::GetSmallPageCapacity<TItem>(2);
- TestListPoolPagesImpl<TItem>(2, count, 2, TListPoolBase::SMALL_MARK);
- }
-
+ ui16 count = TListPoolBase::GetSmallPageCapacity<TItem>(2);
+ TestListPoolPagesImpl<TItem>(2, count, 2, TListPoolBase::SMALL_MARK);
+ }
+
Y_UNIT_TEST(TestListPoolMediumPagesObj) {
- size_t listSize = TListPoolBase::MAX_SMALL_LIST_SIZE + 1;
- ui16 count = TListPoolBase::GetMediumPageCapacity<TItem>(listSize);
- TestListPoolPagesImpl<TItem>(listSize, count, FastClp2(listSize), TListPoolBase::MEDIUM_MARK);
- }
-
+ size_t listSize = TListPoolBase::MAX_SMALL_LIST_SIZE + 1;
+ ui16 count = TListPoolBase::GetMediumPageCapacity<TItem>(listSize);
+ TestListPoolPagesImpl<TItem>(listSize, count, FastClp2(listSize), TListPoolBase::MEDIUM_MARK);
+ }
+
Y_UNIT_TEST(TestListPoolLargPagesObj) {
- TestListPoolLargeImpl<TItem>();
- }
-
- struct TItemHash {
- template <typename T>
- size_t operator() (T num) const {
- return num % 13;
- }
- };
-
- template <typename TItem>
- void TestHashImpl() {
- const ui32 elementsCount = 32;
- const ui64 sumKeysTarget = elementsCount * (elementsCount - 1) / 2;
-
- const ui32 addition = 20;
- const ui64 sumValuesTarget = sumKeysTarget + addition * elementsCount;
-
- TAlignedPagePool pagePool;
+ TestListPoolLargeImpl<TItem>();
+ }
+
+ struct TItemHash {
+ template <typename T>
+ size_t operator() (T num) const {
+ return num % 13;
+ }
+ };
+
+ template <typename TItem>
+ void TestHashImpl() {
+ const ui32 elementsCount = 32;
+ const ui64 sumKeysTarget = elementsCount * (elementsCount - 1) / 2;
+
+ const ui32 addition = 20;
+ const ui64 sumValuesTarget = sumKeysTarget + addition * elementsCount;
+
+ TAlignedPagePool pagePool;
TCompactHash<TItem, TItem, TItemHash> hash(pagePool);
-
+
TVector<TItem> elements(elementsCount);
- std::iota(elements.begin(), elements.end(), 0);
- Shuffle(elements.begin(), elements.end());
- for (TItem i: elements) {
- hash.Insert(i, i + addition);
- }
-
- {
- decltype(hash) hash2(std::move(hash));
+ std::iota(elements.begin(), elements.end(), 0);
+ Shuffle(elements.begin(), elements.end());
+ for (TItem i: elements) {
+ hash.Insert(i, i + addition);
+ }
+
+ {
+ decltype(hash) hash2(std::move(hash));
decltype(hash) hash3(pagePool);
- hash3.Swap(hash2);
- hash = hash3;
- }
-
- for (TItem i: elements) {
- UNIT_ASSERT(hash.Has(i));
- UNIT_ASSERT(hash.Find(i).Ok());
- UNIT_ASSERT_VALUES_EQUAL(i + addition, hash.Find(i).Get().second);
- }
-
- UNIT_ASSERT(!hash.Has(elementsCount + 1));
- UNIT_ASSERT(!hash.Has(elementsCount + 10));
- UNIT_ASSERT_VALUES_EQUAL(elementsCount, hash.Size());
- UNIT_ASSERT_VALUES_EQUAL(elementsCount, hash.UniqSize());
-
- ui64 sumKeys = 0;
- ui64 sumValues = 0;
- for (auto it = hash.Iterate(); it.Ok(); ++it) {
- UNIT_ASSERT_VALUES_EQUAL(it.Get().first + addition, it.Get().second);
-
- sumKeys += it.Get().first;
- sumValues += it.Get().second;
- }
- UNIT_ASSERT_VALUES_EQUAL(sumKeys, sumKeysTarget);
- UNIT_ASSERT_VALUES_EQUAL(sumValues, sumValuesTarget);
- }
-
- template <typename TItem>
- void TestMultiHashImpl() {
- const ui32 keysCount = 10;
- const ui64 elementsCount = keysCount * (keysCount + 1) / 2;
-
- TAlignedPagePool pagePool;
+ hash3.Swap(hash2);
+ hash = hash3;
+ }
+
+ for (TItem i: elements) {
+ UNIT_ASSERT(hash.Has(i));
+ UNIT_ASSERT(hash.Find(i).Ok());
+ UNIT_ASSERT_VALUES_EQUAL(i + addition, hash.Find(i).Get().second);
+ }
+
+ UNIT_ASSERT(!hash.Has(elementsCount + 1));
+ UNIT_ASSERT(!hash.Has(elementsCount + 10));
+ UNIT_ASSERT_VALUES_EQUAL(elementsCount, hash.Size());
+ UNIT_ASSERT_VALUES_EQUAL(elementsCount, hash.UniqSize());
+
+ ui64 sumKeys = 0;
+ ui64 sumValues = 0;
+ for (auto it = hash.Iterate(); it.Ok(); ++it) {
+ UNIT_ASSERT_VALUES_EQUAL(it.Get().first + addition, it.Get().second);
+
+ sumKeys += it.Get().first;
+ sumValues += it.Get().second;
+ }
+ UNIT_ASSERT_VALUES_EQUAL(sumKeys, sumKeysTarget);
+ UNIT_ASSERT_VALUES_EQUAL(sumValues, sumValuesTarget);
+ }
+
+ template <typename TItem>
+ void TestMultiHashImpl() {
+ const ui32 keysCount = 10;
+ const ui64 elementsCount = keysCount * (keysCount + 1) / 2;
+
+ TAlignedPagePool pagePool;
TCompactMultiHash<TItem, TItem, TItemHash> hash(pagePool);
-
+
TVector<TItem> keys(keysCount);
- std::iota(keys.begin(), keys.end(), 0);
- Shuffle(keys.begin(), keys.end());
-
- ui64 sumKeysTarget = 0;
- ui64 sumValuesTarget = 0;
- for (TItem k: keys) {
- sumKeysTarget += k;
- for (TItem i = 0; i < k + 1; ++i) {
- hash.Insert(k, i);
- sumValuesTarget += i;
- }
- }
-
- {
- decltype(hash) hash2(std::move(hash));
+ std::iota(keys.begin(), keys.end(), 0);
+ Shuffle(keys.begin(), keys.end());
+
+ ui64 sumKeysTarget = 0;
+ ui64 sumValuesTarget = 0;
+ for (TItem k: keys) {
+ sumKeysTarget += k;
+ for (TItem i = 0; i < k + 1; ++i) {
+ hash.Insert(k, i);
+ sumValuesTarget += i;
+ }
+ }
+
+ {
+ decltype(hash) hash2(std::move(hash));
decltype(hash) hash3(pagePool);
- hash3.Swap(hash2);
- hash = hash3;
- }
-
- for (TItem k: keys) {
- UNIT_ASSERT(hash.Has(k));
- UNIT_ASSERT_VALUES_EQUAL(k + 1, hash.Count(k));
- auto it = hash.Find(k);
- UNIT_ASSERT(it.Ok());
- TDynBitMap set;
- for (; it.Ok(); ++it) {
- UNIT_ASSERT_VALUES_EQUAL(k, it.GetKey());
- UNIT_ASSERT(!set.Test(it.GetValue()));
- set.Set(it.GetValue());
- }
- UNIT_ASSERT_VALUES_EQUAL(set.Count(), k + 1);
- }
-
- UNIT_ASSERT(!hash.Has(keysCount + 1));
- UNIT_ASSERT(!hash.Has(keysCount + 10));
- UNIT_ASSERT(!hash.Find(keysCount + 1).Ok());
-
- UNIT_ASSERT_VALUES_EQUAL(elementsCount, hash.Size());
- UNIT_ASSERT_VALUES_EQUAL(keysCount, hash.UniqSize());
-
- ui64 sumKeys = 0;
- ui64 sumValues = 0;
- TMaybe<TItem> prevKey;
- for (auto it = hash.Iterate(); it.Ok(); ++it) {
- const auto key = it.GetKey();
- if (prevKey != key) {
- sumKeys += key;
- for (auto valIt = it.MakeCurrentKeyIter(); valIt.Ok(); ++valIt) {
- UNIT_ASSERT_VALUES_EQUAL(key, valIt.GetKey());
- sumValues += valIt.GetValue();
- }
- prevKey = key;
- }
- }
- UNIT_ASSERT_VALUES_EQUAL(sumKeys, sumKeysTarget);
- UNIT_ASSERT_VALUES_EQUAL(sumValues, sumValuesTarget);
-
- // Test large lists
- TItem val = 0;
- for (size_t i = 0; i < TListPoolBase::GetLargePageCapacity<TItem>(); ++i) {
- hash.Insert(keysCount, val++);
- }
-
- TItem check = 0;
- for (auto i = hash.Find(keysCount); i.Ok(); ++i) {
- UNIT_ASSERT_VALUES_EQUAL(keysCount, i.GetKey());
- UNIT_ASSERT_VALUES_EQUAL(check++, i.GetValue());
- }
- UNIT_ASSERT_VALUES_EQUAL(check, val);
- UNIT_ASSERT_VALUES_EQUAL(TListPoolBase::GetLargePageCapacity<TItem>(), hash.Count(keysCount));
-
- for (size_t i = 0; i < TListPoolBase::GetLargePageCapacity<TItem>() + 1; ++i) {
- hash.Insert(keysCount, val++);
- }
-
- {
- decltype(hash) hash2(std::move(hash));
+ hash3.Swap(hash2);
+ hash = hash3;
+ }
+
+ for (TItem k: keys) {
+ UNIT_ASSERT(hash.Has(k));
+ UNIT_ASSERT_VALUES_EQUAL(k + 1, hash.Count(k));
+ auto it = hash.Find(k);
+ UNIT_ASSERT(it.Ok());
+ TDynBitMap set;
+ for (; it.Ok(); ++it) {
+ UNIT_ASSERT_VALUES_EQUAL(k, it.GetKey());
+ UNIT_ASSERT(!set.Test(it.GetValue()));
+ set.Set(it.GetValue());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(set.Count(), k + 1);
+ }
+
+ UNIT_ASSERT(!hash.Has(keysCount + 1));
+ UNIT_ASSERT(!hash.Has(keysCount + 10));
+ UNIT_ASSERT(!hash.Find(keysCount + 1).Ok());
+
+ UNIT_ASSERT_VALUES_EQUAL(elementsCount, hash.Size());
+ UNIT_ASSERT_VALUES_EQUAL(keysCount, hash.UniqSize());
+
+ ui64 sumKeys = 0;
+ ui64 sumValues = 0;
+ TMaybe<TItem> prevKey;
+ for (auto it = hash.Iterate(); it.Ok(); ++it) {
+ const auto key = it.GetKey();
+ if (prevKey != key) {
+ sumKeys += key;
+ for (auto valIt = it.MakeCurrentKeyIter(); valIt.Ok(); ++valIt) {
+ UNIT_ASSERT_VALUES_EQUAL(key, valIt.GetKey());
+ sumValues += valIt.GetValue();
+ }
+ prevKey = key;
+ }
+ }
+ UNIT_ASSERT_VALUES_EQUAL(sumKeys, sumKeysTarget);
+ UNIT_ASSERT_VALUES_EQUAL(sumValues, sumValuesTarget);
+
+ // Test large lists
+ TItem val = 0;
+ for (size_t i = 0; i < TListPoolBase::GetLargePageCapacity<TItem>(); ++i) {
+ hash.Insert(keysCount, val++);
+ }
+
+ TItem check = 0;
+ for (auto i = hash.Find(keysCount); i.Ok(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(keysCount, i.GetKey());
+ UNIT_ASSERT_VALUES_EQUAL(check++, i.GetValue());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(check, val);
+ UNIT_ASSERT_VALUES_EQUAL(TListPoolBase::GetLargePageCapacity<TItem>(), hash.Count(keysCount));
+
+ for (size_t i = 0; i < TListPoolBase::GetLargePageCapacity<TItem>() + 1; ++i) {
+ hash.Insert(keysCount, val++);
+ }
+
+ {
+ decltype(hash) hash2(std::move(hash));
decltype(hash) hash3(pagePool);
- hash3.Swap(hash2);
- hash = hash3;
- }
-
- check = 0;
- for (auto i = hash.Find(keysCount); i.Ok(); ++i) {
- UNIT_ASSERT_VALUES_EQUAL(keysCount, i.GetKey());
- UNIT_ASSERT_VALUES_EQUAL(check++, i.GetValue());
- }
- UNIT_ASSERT_VALUES_EQUAL(check, val);
- UNIT_ASSERT_VALUES_EQUAL(2 * TListPoolBase::GetLargePageCapacity<TItem>() + 1, hash.Count(keysCount));
- }
-
- template <typename TItem>
- void TestSetImpl() {
- const ui32 elementsCount = 32;
- const ui64 sumKeysTarget = elementsCount * (elementsCount - 1) / 2;
-
- TAlignedPagePool pagePool;
+ hash3.Swap(hash2);
+ hash = hash3;
+ }
+
+ check = 0;
+ for (auto i = hash.Find(keysCount); i.Ok(); ++i) {
+ UNIT_ASSERT_VALUES_EQUAL(keysCount, i.GetKey());
+ UNIT_ASSERT_VALUES_EQUAL(check++, i.GetValue());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(check, val);
+ UNIT_ASSERT_VALUES_EQUAL(2 * TListPoolBase::GetLargePageCapacity<TItem>() + 1, hash.Count(keysCount));
+ }
+
+ template <typename TItem>
+ void TestSetImpl() {
+ const ui32 elementsCount = 32;
+ const ui64 sumKeysTarget = elementsCount * (elementsCount - 1) / 2;
+
+ TAlignedPagePool pagePool;
TCompactHashSet<TItem, TItemHash> hash(pagePool);
-
+
TVector<TItem> elements(elementsCount);
- std::iota(elements.begin(), elements.end(), 0);
- Shuffle(elements.begin(), elements.end());
- for (TItem i: elements) {
- hash.Insert(i);
- }
-
- {
- decltype(hash) hash2(std::move(hash));
+ std::iota(elements.begin(), elements.end(), 0);
+ Shuffle(elements.begin(), elements.end());
+ for (TItem i: elements) {
+ hash.Insert(i);
+ }
+
+ {
+ decltype(hash) hash2(std::move(hash));
decltype(hash) hash3(pagePool);
- hash3.Swap(hash2);
- hash = hash3;
- }
-
- for (TItem i: elements) {
- UNIT_ASSERT(hash.Has(i));
- }
-
- UNIT_ASSERT(!hash.Has(elementsCount + 1));
- UNIT_ASSERT(!hash.Has(elementsCount + 10));
- UNIT_ASSERT_VALUES_EQUAL(elementsCount, hash.Size());
- UNIT_ASSERT_VALUES_EQUAL(elementsCount, hash.UniqSize());
-
- ui64 sumKeys = 0;
- for (auto i = hash.Iterate(); i.Ok(); ++i) {
- sumKeys += *i;
- }
- UNIT_ASSERT_VALUES_EQUAL(sumKeys, sumKeysTarget);
- }
-
+ hash3.Swap(hash2);
+ hash = hash3;
+ }
+
+ for (TItem i: elements) {
+ UNIT_ASSERT(hash.Has(i));
+ }
+
+ UNIT_ASSERT(!hash.Has(elementsCount + 1));
+ UNIT_ASSERT(!hash.Has(elementsCount + 10));
+ UNIT_ASSERT_VALUES_EQUAL(elementsCount, hash.Size());
+ UNIT_ASSERT_VALUES_EQUAL(elementsCount, hash.UniqSize());
+
+ ui64 sumKeys = 0;
+ for (auto i = hash.Iterate(); i.Ok(); ++i) {
+ sumKeys += *i;
+ }
+ UNIT_ASSERT_VALUES_EQUAL(sumKeys, sumKeysTarget);
+ }
+
Y_UNIT_TEST(TestHashByte) {
- TestHashImpl<ui8>();
- }
-
+ TestHashImpl<ui8>();
+ }
+
Y_UNIT_TEST(TestMultiHashByte) {
- TestMultiHashImpl<ui8>();
- }
-
+ TestMultiHashImpl<ui8>();
+ }
+
Y_UNIT_TEST(TestSetByte) {
- TestSetImpl<ui8>();
- }
-
+ TestSetImpl<ui8>();
+ }
+
Y_UNIT_TEST(TestHashUi16) {
- TestHashImpl<ui16>();
- }
-
+ TestHashImpl<ui16>();
+ }
+
Y_UNIT_TEST(TestMultiHashUi16) {
- TestMultiHashImpl<ui16>();
- }
-
+ TestMultiHashImpl<ui16>();
+ }
+
Y_UNIT_TEST(TestSetUi16) {
- TestSetImpl<ui16>();
- }
-
+ TestSetImpl<ui16>();
+ }
+
Y_UNIT_TEST(TestHashUi64) {
- TestHashImpl<ui64>();
- }
-
+ TestHashImpl<ui64>();
+ }
+
Y_UNIT_TEST(TestMultiHashUi64) {
- TestMultiHashImpl<ui64>();
- }
-
+ TestMultiHashImpl<ui64>();
+ }
+
Y_UNIT_TEST(TestSetUi64) {
- TestSetImpl<ui64>();
- }
-
- struct TStressHash {
- TStressHash(size_t param)
- : Param(param)
- {
- }
-
- template <typename T>
- size_t operator() (T num) const {
- return num % Param;
- }
- const size_t Param;
- };
-
- Y_UNIT_TEST(TestStressSmallLists) {
- TAlignedPagePool pagePool;
- for (size_t listSize: xrange<size_t>(2, 17, 1)) {
- const size_t backets = TListPoolBase::GetSmallPageCapacity<ui64>(listSize);
- const size_t elementsCount = backets * listSize;
-
- for (size_t count: xrange<size_t>(1, elementsCount + 1, elementsCount / 16)) {
- TCompactHashSet<ui64, TStressHash> hash(pagePool, elementsCount, TStressHash(backets));
- for (auto i: xrange(count)) {
- hash.Insert(i);
- }
- UNIT_ASSERT_VALUES_EQUAL(count, hash.Size());
- UNIT_ASSERT_VALUES_EQUAL(count, hash.UniqSize());
- //hash.PrintStat(Cerr);
- }
- }
- }
-}
+ TestSetImpl<ui64>();
+ }
+
+ struct TStressHash {
+ TStressHash(size_t param)
+ : Param(param)
+ {
+ }
+
+ template <typename T>
+ size_t operator() (T num) const {
+ return num % Param;
+ }
+ const size_t Param;
+ };
+
+ Y_UNIT_TEST(TestStressSmallLists) {
+ TAlignedPagePool pagePool;
+ for (size_t listSize: xrange<size_t>(2, 17, 1)) {
+ const size_t backets = TListPoolBase::GetSmallPageCapacity<ui64>(listSize);
+ const size_t elementsCount = backets * listSize;
+
+ for (size_t count: xrange<size_t>(1, elementsCount + 1, elementsCount / 16)) {
+ TCompactHashSet<ui64, TStressHash> hash(pagePool, elementsCount, TStressHash(backets));
+ for (auto i: xrange(count)) {
+ hash.Insert(i);
+ }
+ UNIT_ASSERT_VALUES_EQUAL(count, hash.Size());
+ UNIT_ASSERT_VALUES_EQUAL(count, hash.UniqSize());
+ //hash.PrintStat(Cerr);
+ }
+ }
+ }
+}
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_list_adapter.h b/ydb/library/yql/minikql/computation/mkql_computation_list_adapter.h
index 2b3b7fb886..1f86ad6e5d 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_list_adapter.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_list_adapter.h
@@ -2,7 +2,7 @@
#include <ydb/library/yql/minikql/defs.h>
#include "mkql_computation_node_impl.h"
#include <ydb/library/yql/minikql/mkql_alloc.h>
-
+
namespace NKikimr {
namespace NMiniKQL {
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node.cpp
index aa2e27f028..364c76cdeb 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node.cpp
@@ -1,9 +1,9 @@
#include "mkql_computation_node_holders.h"
#include "mkql_computation_node_impl.h"
#include "mkql_computation_node_pack.h"
-#include "mkql_value_builder.h"
-#include "mkql_validate.h"
-
+#include "mkql_value_builder.h"
+#include "mkql_validate.h"
+
#include <ydb/library/yql/minikql/mkql_node_builder.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_node_printer.h>
@@ -15,12 +15,12 @@
#include <util/generic/set.h>
#include <util/generic/algorithm.h>
#include <util/random/mersenne.h>
-#include <util/random/random.h>
+#include <util/random/random.h>
#include <util/system/tempfile.h>
-#include <util/system/fstat.h>
-#include <util/system/rusage.h>
+#include <util/system/fstat.h>
+#include <util/system/rusage.h>
#include <util/stream/file.h>
-#include <util/stream/output.h>
+#include <util/stream/output.h>
#include <util/memory/pool.h>
namespace NKikimr {
@@ -35,56 +35,56 @@ TComputationContext::TComputationContext(const THolderFactory& holderFactory,
, RandomProvider(opts.RandomProvider)
, TimeProvider(opts.TimeProvider)
, ArrowMemoryPool(arrowMemoryPool)
-{
+{
std::fill_n(MutableValues.get(), mutables.CurValueIndex, NUdf::TUnboxedValue(NUdf::TUnboxedValuePod::Invalid()));
-}
-
-TComputationContext::~TComputationContext() {
-#ifndef NDEBUG
+}
+
+TComputationContext::~TComputationContext() {
+#ifndef NDEBUG
if (RssCounter) {
- Cerr << "UsageOnFinish: graph=" << HolderFactory.GetPagePool().GetUsed()
+ Cerr << "UsageOnFinish: graph=" << HolderFactory.GetPagePool().GetUsed()
<< ", rss=" << TRusage::Get().MaxRss
- << ", peakAlloc=" << HolderFactory.GetPagePool().GetPeakAllocated()
- << ", adjustor=" << UsageAdjustor
- << Endl;
- }
-#endif
-}
-
-void TComputationContext::UpdateUsageAdjustor(ui64 memLimit) {
+ << ", peakAlloc=" << HolderFactory.GetPagePool().GetPeakAllocated()
+ << ", adjustor=" << UsageAdjustor
+ << Endl;
+ }
+#endif
+}
+
+void TComputationContext::UpdateUsageAdjustor(ui64 memLimit) {
const auto rss = TRusage::Get().MaxRss;
- if (!InitRss) {
- LastRss = InitRss = rss;
- }
-
-#ifndef NDEBUG
+ if (!InitRss) {
+ LastRss = InitRss = rss;
+ }
+
+#ifndef NDEBUG
// Print first time and then each 30 seconds
bool printUsage = LastPrintUsage == TInstant::Zero()
|| TInstant::Now() > TDuration::Seconds(30).ToDeadLine(LastPrintUsage);
-#endif
-
- if (auto peakAlloc = HolderFactory.GetPagePool().GetPeakAllocated()) {
- if (rss - InitRss > memLimit && rss - LastRss > (memLimit / 4)) {
+#endif
+
+ if (auto peakAlloc = HolderFactory.GetPagePool().GetPeakAllocated()) {
+ if (rss - InitRss > memLimit && rss - LastRss > (memLimit / 4)) {
UsageAdjustor = std::max(1.f, float(rss - InitRss) / float(peakAlloc));
- LastRss = rss;
-#ifndef NDEBUG
+ LastRss = rss;
+#ifndef NDEBUG
printUsage = UsageAdjustor > 1.f;
-#endif
- }
+#endif
+ }
}
-
-#ifndef NDEBUG
+
+#ifndef NDEBUG
if (printUsage) {
- Cerr << "Usage: graph=" << HolderFactory.GetPagePool().GetUsed()
+ Cerr << "Usage: graph=" << HolderFactory.GetPagePool().GetUsed()
<< ", rss=" << rss
<< ", peakAlloc=" << HolderFactory.GetPagePool().GetPeakAllocated()
<< ", adjustor=" << UsageAdjustor
<< Endl;
LastPrintUsage = TInstant::Now();
}
-#endif
-}
-
+#endif
+}
+
class TSimpleSecureParamsProvider : public NUdf::ISecureParamsProvider {
public:
TSimpleSecureParamsProvider(const THashMap<TString, TString>& secureParams)
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node.h b/ydb/library/yql/minikql/computation/mkql_computation_node.h
index a5d11b180b..8e6dc1129b 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node.h
@@ -1,7 +1,7 @@
#pragma once
-#include "mkql_computation_node_list.h"
-
+#include "mkql_computation_node_list.h"
+
#include <ydb/library/yql/minikql/defs.h>
#include <ydb/library/yql/minikql/arrow/mkql_memory_pool.h>
#include <ydb/library/yql/minikql/mkql_node.h>
@@ -14,7 +14,7 @@
#include <ydb/library/yql/public/udf/udf_value.h>
#include <ydb/library/yql/public/udf/udf_validate.h>
#include <ydb/library/yql/public/udf/udf_value_builder.h>
-
+
#include <library/cpp/random_provider/random_provider.h>
#include <library/cpp/time_provider/time_provider.h>
@@ -30,11 +30,11 @@ inline const TDefaultListRepresentation* GetDefaultListRepresentation(const NUdf
return reinterpret_cast<const TDefaultListRepresentation*>(NUdf::TBoxedValueAccessor::GetListRepresentation(*value.AsBoxed()));
}
-enum class EGraphPerProcess {
- Multi,
- Single
-};
-
+enum class EGraphPerProcess {
+ Multi,
+ Single
+};
+
struct TComputationOpts {
TComputationOpts(IStatsRegistry* stats)
: Stats(stats)
@@ -89,20 +89,20 @@ struct TComputationContext : public TComputationContextLLVM {
TComputationOptsFull& opts,
const TComputationMutables& mutables,
arrow::MemoryPool& arrowMemoryPool);
- ~TComputationContext();
-
- // Returns true if current usage delta exceeds the memory limit
- // The function automatically adjusts memory limit taking into account RSS delta between calls
- template<bool TrackRss>
+ ~TComputationContext();
+
+ // Returns true if current usage delta exceeds the memory limit
+ // The function automatically adjusts memory limit taking into account RSS delta between calls
+ template<bool TrackRss>
inline bool CheckAdjustedMemLimit(ui64 memLimit, ui64 initMemUsage);
-
+
void UpdateUsageAdjustor(ui64 memLimit);
-private:
+private:
ui64 InitRss = 0ULL;
ui64 LastRss = 0ULL;
-#ifndef NDEBUG
- TInstant LastPrintUsage;
-#endif
+#ifndef NDEBUG
+ TInstant LastPrintUsage;
+#endif
};
class IComputationNode {
@@ -209,9 +209,9 @@ struct TComputationNodeFactoryContext {
const TNodeFactory& NodeFactory;
const THolderFactory& HolderFactory;
const NUdf::IValueBuilder *const Builder;
- NUdf::EValidateMode ValidateMode;
- NUdf::EValidatePolicy ValidatePolicy;
- EGraphPerProcess GraphPerProcess;
+ NUdf::EValidateMode ValidateMode;
+ NUdf::EValidatePolicy ValidatePolicy;
+ EGraphPerProcess GraphPerProcess;
TComputationMutables& Mutables;
TComputationNodeOnNodeMap& ElementsCache;
const TNodePushBack NodePushBack;
@@ -226,13 +226,13 @@ struct TComputationNodeFactoryContext {
const TNodeFactory& nodeFactory,
const THolderFactory& holderFactory,
const NUdf::IValueBuilder* builder,
- NUdf::EValidateMode validateMode,
- NUdf::EValidatePolicy validatePolicy,
+ NUdf::EValidateMode validateMode,
+ NUdf::EValidatePolicy validatePolicy,
EGraphPerProcess graphPerProcess,
TComputationMutables& mutables,
TComputationNodeOnNodeMap& elementsCache,
TNodePushBack&& nodePushBack
- )
+ )
: NodeLocator(nodeLocator)
, FunctionRegistry(functionRegistry)
, Env(env)
@@ -242,9 +242,9 @@ struct TComputationNodeFactoryContext {
, NodeFactory(nodeFactory)
, HolderFactory(holderFactory)
, Builder(builder)
- , ValidateMode(validateMode)
- , ValidatePolicy(validatePolicy)
- , GraphPerProcess(graphPerProcess)
+ , ValidateMode(validateMode)
+ , ValidatePolicy(validatePolicy)
+ , GraphPerProcess(graphPerProcess)
, Mutables(mutables)
, ElementsCache(elementsCache)
, NodePushBack(std::move(nodePushBack))
@@ -272,21 +272,21 @@ struct TComputationPatternOpts {
const TTypeEnvironment& env,
TComputationNodeFactory factory,
const IFunctionRegistry* functionRegistry,
- NUdf::EValidateMode validateMode,
- NUdf::EValidatePolicy validatePolicy,
+ NUdf::EValidateMode validateMode,
+ NUdf::EValidatePolicy validatePolicy,
const TString& optLLVM,
- EGraphPerProcess graphPerProcess,
+ EGraphPerProcess graphPerProcess,
IStatsRegistry* stats = nullptr,
NUdf::ICountersProvider* countersProvider = nullptr)
- : AllocState(allocState)
- , Env(env)
- , Factory(factory)
- , FunctionRegistry(functionRegistry)
- , ValidateMode(validateMode)
- , ValidatePolicy(validatePolicy)
+ : AllocState(allocState)
+ , Env(env)
+ , Factory(factory)
+ , FunctionRegistry(functionRegistry)
+ , ValidateMode(validateMode)
+ , ValidatePolicy(validatePolicy)
, OptLLVM(optLLVM)
- , GraphPerProcess(graphPerProcess)
- , Stats(stats)
+ , GraphPerProcess(graphPerProcess)
+ , Stats(stats)
, CountersProvider(countersProvider)
{}
@@ -303,14 +303,14 @@ struct TComputationPatternOpts {
Stats = stats;
CountersProvider = counters;
SecureParamsProvider = secureParamsProvider;
- }
+ }
mutable std::shared_ptr<TInjectedAlloc> CacheAlloc;
mutable std::shared_ptr<TTypeEnvironment> CacheEnv;
- TAllocState& AllocState;
- const TTypeEnvironment& Env;
+ TAllocState& AllocState;
+ const TTypeEnvironment& Env;
- TComputationNodeFactory Factory;
+ TComputationNodeFactory Factory;
const IFunctionRegistry* FunctionRegistry = nullptr;
NUdf::EValidateMode ValidateMode = NUdf::EValidateMode::None;
NUdf::EValidatePolicy ValidatePolicy = NUdf::EValidatePolicy::Fail;
@@ -324,7 +324,7 @@ struct TComputationPatternOpts {
TComputationOptsFull ToComputationOptions(IRandomProvider& randomProvider, ITimeProvider& timeProvider, TAllocState* allocStatePtr = nullptr) const {
return TComputationOptsFull(Stats, allocStatePtr ? *allocStatePtr : AllocState, randomProvider, timeProvider, ValidatePolicy, SecureParamsProvider);
}
-};
+};
class IComputationPattern: public TAtomicRefCount<IComputationPattern> {
public:
@@ -334,13 +334,13 @@ public:
virtual THolder<IComputationGraph> Clone(const TComputationOptsFull& compOpts) = 0;
};
-// node cookie's will be clean up when graph will be destroyed, explorer must not be changed/destroyed until that time
+// node cookie's will be clean up when graph will be destroyed, explorer must not be changed/destroyed until that time
IComputationPattern::TPtr MakeComputationPattern(
- TExploringNodeVisitor& explorer,
+ TExploringNodeVisitor& explorer,
const TRuntimeNode& root,
const std::vector<TNode*>& entryPoints,
const TComputationPatternOpts& opts);
-
+
class IComputationPatternCache {
public:
typedef THolder<IComputationPatternCache> TPtr;
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp
index 563f820bf6..315ef3950b 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.cpp
@@ -795,10 +795,10 @@ Value* TUnboxedImmutableCodegeneratorNode::CreateGetValue(const TCodegenContext&
return ConstantInt::get(Type::getInt128Ty(ctx.Codegen->GetContext()), APInt(128, 2, reinterpret_cast<const uint64_t*>(&UnboxedValue)));
}
-TUnboxedImmutableRunCodegeneratorNode::TUnboxedImmutableRunCodegeneratorNode(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& value)
- : TUnboxedImmutableComputationNode(memInfo, std::move(value))
-{}
-
+TUnboxedImmutableRunCodegeneratorNode::TUnboxedImmutableRunCodegeneratorNode(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& value)
+ : TUnboxedImmutableComputationNode(memInfo, std::move(value))
+{}
+
TExternalCodegeneratorNode::TExternalCodegeneratorNode(TComputationMutables& mutables, EValueRepresentation kind)
: TExternalComputationNode(mutables, kind)
{}
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h b/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h
index d9feede204..a062f97968 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_codegen.h
@@ -11,7 +11,7 @@ namespace NKikimr {
namespace NMiniKQL {
using TUnboxedImmutableCodegeneratorNode = TUnboxedImmutableComputationNode;
-using TUnboxedImmutableRunCodegeneratorNode = TUnboxedImmutableComputationNode;
+using TUnboxedImmutableRunCodegeneratorNode = TUnboxedImmutableComputationNode;
using TExternalCodegeneratorNode = TExternalComputationNode;
using TWideFlowProxyCodegeneratorNode = TWideFlowProxyComputationNode;
@@ -106,14 +106,14 @@ using TGeneratorPtr = Value* (*)(Value *const * args, const TCodegenContext& ctx
class ICodegeneratorRootNode {
public:
- virtual ~ICodegeneratorRootNode() {}
+ virtual ~ICodegeneratorRootNode() {}
virtual void GenerateFunctions(const NYql::NCodegen::ICodegen::TPtr& codegen) = 0;
virtual void FinalizeFunctions(const NYql::NCodegen::ICodegen::TPtr& codegen) = 0;
};
class ICodegeneratorInlineNode {
public:
- virtual ~ICodegeneratorInlineNode() {}
+ virtual ~ICodegeneratorInlineNode() {}
virtual Value* CreateGetValue(const TCodegenContext& ctx, BasicBlock*& block) const = 0;
};
@@ -147,12 +147,12 @@ public:
virtual void SetValueGetter(Function* value) = 0;
};
-class ICodegeneratorRunNode {
-public:
- virtual ~ICodegeneratorRunNode() {}
+class ICodegeneratorRunNode {
+public:
+ virtual ~ICodegeneratorRunNode() {}
virtual void CreateRun(const TCodegenContext& ctx, BasicBlock*& block, Value* result, Value* args) const = 0;
-};
-
+};
+
size_t GetMethodPtrIndex(uintptr_t ptr);
template<typename Method>
@@ -918,12 +918,12 @@ private:
Value* CreateGetValue(const TCodegenContext& ctx, BasicBlock*&) const final;
};
-class TUnboxedImmutableRunCodegeneratorNode: public TUnboxedImmutableComputationNode, public ICodegeneratorRunNode
-{
-public:
- TUnboxedImmutableRunCodegeneratorNode(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& value);
-};
-
+class TUnboxedImmutableRunCodegeneratorNode: public TUnboxedImmutableComputationNode, public ICodegeneratorRunNode
+{
+public:
+ TUnboxedImmutableRunCodegeneratorNode(TMemoryUsageInfo* memInfo, NUdf::TUnboxedValue&& value);
+};
+
class TExternalCodegeneratorNode: public TExternalComputationNode, public ICodegeneratorExternalNode
{
public:
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp
index 59402e06dd..e9e71a6aa0 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_dict_ut.cpp
@@ -6,119 +6,119 @@
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
#include <ydb/library/yql/minikql/comp_nodes/mkql_factories.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <vector>
-#include <utility>
-#include <algorithm>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-namespace {
-struct TSetup {
- TSetup(TScopedAlloc& alloc)
- : Alloc(alloc)
- {
- FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
- RandomProvider = CreateDeterministicRandomProvider(1);
- TimeProvider = CreateDeterministicTimeProvider(10000000);
-
- Env.Reset(new TTypeEnvironment(Alloc));
- PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
- }
-
- THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
- Explorer.Walk(pgm.GetNode(), *Env);
- TComputationPatternOpts opts(Alloc.Ref(), *Env, GetBuiltinFactory(),
- FunctionRegistry.Get(),
- NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, "OFF", EGraphPerProcess::Multi);
- Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
- TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
- return Pattern->Clone(compOpts);
- }
-
- TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
- TIntrusivePtr<IRandomProvider> RandomProvider;
- TIntrusivePtr<ITimeProvider> TimeProvider;
-
- TScopedAlloc& Alloc;
- THolder<TTypeEnvironment> Env;
- THolder<TProgramBuilder> PgmBuilder;
-
- TExploringNodeVisitor Explorer;
- IComputationPattern::TPtr Pattern;
-};
-}
-
-Y_UNIT_TEST_SUITE(TestCompactMultiDict) {
- Y_UNIT_TEST(TestIterate) {
- TScopedAlloc alloc;
-
- TSetup setup(alloc);
-
- const std::vector<std::pair<ui32, std::vector<ui32>>> items = {{1, {1, 2}}, {2, {1}}, {3, {0}}, {6, {1, 7}}};
-
- TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
- TVector<TRuntimeNode> rItems;
- for (auto& [k, vv]: items) {
- for (auto& v: vv) {
- rItems.push_back(pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<ui32>(k), pgmBuilder.NewDataLiteral<ui32>(v)}));
- }
- }
- auto ui32Type = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
- auto list = pgmBuilder.NewList(pgmBuilder.NewTupleType({ui32Type, ui32Type}), rItems);
-
- auto dict = pgmBuilder.ToHashedDict(list, /*all*/true,
- [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.Nth(item, 0); },
- [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.Nth(item, 1); },
- /*isCompact*/true,
- items.size());
-
- auto graph = setup.BuildGraph(dict, {});
- NUdf::TUnboxedValue res = graph->GetValue();
-
- std::vector<ui32> keyVals;
- for (NUdf::TUnboxedValue keys = res.GetKeysIterator(), v; keys.Next(v);) {
- keyVals.push_back(v.Get<ui32>());
- }
- UNIT_ASSERT_VALUES_EQUAL(keyVals.size(), items.size());
- std::sort(keyVals.begin(), keyVals.end());
- UNIT_ASSERT(
- std::equal(keyVals.begin(), keyVals.end(), items.begin(),
- [](ui32 l, const std::pair<ui32, std::vector<ui32>>& r) { return l == r.first; }
- )
- );
-
- std::vector<std::vector<ui32>> origPayloads;
- for (auto& [k, vv]: items) {
- origPayloads.push_back(vv);
- std::sort(origPayloads.back().begin(), origPayloads.back().end());
- }
- std::sort(origPayloads.begin(), origPayloads.end());
-
- std::vector<std::vector<ui32>> payloadVals;
- for (NUdf::TUnboxedValue payloads = res.GetPayloadsIterator(), v; payloads.Next(v);) {
- payloadVals.emplace_back();
- for (NUdf::TUnboxedValue i = v.GetListIterator(), p; i.Next(p);) {
- payloadVals.back().push_back(p.Get<ui32>());
- }
- std::sort(payloadVals.back().begin(), payloadVals.back().end());
- }
- std::sort(payloadVals.begin(), payloadVals.end());
- UNIT_ASSERT_VALUES_EQUAL(origPayloads, payloadVals);
-
- std::vector<std::pair<ui32, std::vector<ui32>>> vals;
- for (NUdf::TUnboxedValue values = res.GetDictIterator(), k, payloads; values.NextPair(k, payloads);) {
- vals.emplace_back(k.Get<ui32>(), std::vector<ui32>{});
- for (NUdf::TUnboxedValue i = payloads.GetListIterator(), p; i.Next(p);) {
- vals.back().second.push_back(p.Get<ui32>());
- }
- std::sort(vals.back().second.begin(), vals.back().second.end());
- }
- UNIT_ASSERT_VALUES_EQUAL(items, vals);
- }
-}
-}
-}
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <vector>
+#include <utility>
+#include <algorithm>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+namespace {
+struct TSetup {
+ TSetup(TScopedAlloc& alloc)
+ : Alloc(alloc)
+ {
+ FunctionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+ RandomProvider = CreateDeterministicRandomProvider(1);
+ TimeProvider = CreateDeterministicTimeProvider(10000000);
+
+ Env.Reset(new TTypeEnvironment(Alloc));
+ PgmBuilder.Reset(new TProgramBuilder(*Env, *FunctionRegistry));
+ }
+
+ THolder<IComputationGraph> BuildGraph(TRuntimeNode pgm, const std::vector<TNode*>& entryPoints = std::vector<TNode*>()) {
+ Explorer.Walk(pgm.GetNode(), *Env);
+ TComputationPatternOpts opts(Alloc.Ref(), *Env, GetBuiltinFactory(),
+ FunctionRegistry.Get(),
+ NUdf::EValidateMode::None, NUdf::EValidatePolicy::Exception, "OFF", EGraphPerProcess::Multi);
+ Pattern = MakeComputationPattern(Explorer, pgm, entryPoints, opts);
+ TComputationOptsFull compOpts = opts.ToComputationOptions(*RandomProvider, *TimeProvider);
+ return Pattern->Clone(compOpts);
+ }
+
+ TIntrusivePtr<IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
+ TIntrusivePtr<ITimeProvider> TimeProvider;
+
+ TScopedAlloc& Alloc;
+ THolder<TTypeEnvironment> Env;
+ THolder<TProgramBuilder> PgmBuilder;
+
+ TExploringNodeVisitor Explorer;
+ IComputationPattern::TPtr Pattern;
+};
+}
+
+Y_UNIT_TEST_SUITE(TestCompactMultiDict) {
+ Y_UNIT_TEST(TestIterate) {
+ TScopedAlloc alloc;
+
+ TSetup setup(alloc);
+
+ const std::vector<std::pair<ui32, std::vector<ui32>>> items = {{1, {1, 2}}, {2, {1}}, {3, {0}}, {6, {1, 7}}};
+
+ TProgramBuilder& pgmBuilder = *setup.PgmBuilder;
+ TVector<TRuntimeNode> rItems;
+ for (auto& [k, vv]: items) {
+ for (auto& v: vv) {
+ rItems.push_back(pgmBuilder.NewTuple({pgmBuilder.NewDataLiteral<ui32>(k), pgmBuilder.NewDataLiteral<ui32>(v)}));
+ }
+ }
+ auto ui32Type = pgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
+ auto list = pgmBuilder.NewList(pgmBuilder.NewTupleType({ui32Type, ui32Type}), rItems);
+
+ auto dict = pgmBuilder.ToHashedDict(list, /*all*/true,
+ [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.Nth(item, 0); },
+ [&pgmBuilder](TRuntimeNode item) { return pgmBuilder.Nth(item, 1); },
+ /*isCompact*/true,
+ items.size());
+
+ auto graph = setup.BuildGraph(dict, {});
+ NUdf::TUnboxedValue res = graph->GetValue();
+
+ std::vector<ui32> keyVals;
+ for (NUdf::TUnboxedValue keys = res.GetKeysIterator(), v; keys.Next(v);) {
+ keyVals.push_back(v.Get<ui32>());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(keyVals.size(), items.size());
+ std::sort(keyVals.begin(), keyVals.end());
+ UNIT_ASSERT(
+ std::equal(keyVals.begin(), keyVals.end(), items.begin(),
+ [](ui32 l, const std::pair<ui32, std::vector<ui32>>& r) { return l == r.first; }
+ )
+ );
+
+ std::vector<std::vector<ui32>> origPayloads;
+ for (auto& [k, vv]: items) {
+ origPayloads.push_back(vv);
+ std::sort(origPayloads.back().begin(), origPayloads.back().end());
+ }
+ std::sort(origPayloads.begin(), origPayloads.end());
+
+ std::vector<std::vector<ui32>> payloadVals;
+ for (NUdf::TUnboxedValue payloads = res.GetPayloadsIterator(), v; payloads.Next(v);) {
+ payloadVals.emplace_back();
+ for (NUdf::TUnboxedValue i = v.GetListIterator(), p; i.Next(p);) {
+ payloadVals.back().push_back(p.Get<ui32>());
+ }
+ std::sort(payloadVals.back().begin(), payloadVals.back().end());
+ }
+ std::sort(payloadVals.begin(), payloadVals.end());
+ UNIT_ASSERT_VALUES_EQUAL(origPayloads, payloadVals);
+
+ std::vector<std::pair<ui32, std::vector<ui32>>> vals;
+ for (NUdf::TUnboxedValue values = res.GetDictIterator(), k, payloads; values.NextPair(k, payloads);) {
+ vals.emplace_back(k.Get<ui32>(), std::vector<ui32>{});
+ for (NUdf::TUnboxedValue i = payloads.GetListIterator(), p; i.Next(p);) {
+ vals.back().second.push_back(p.Get<ui32>());
+ }
+ std::sort(vals.back().second.begin(), vals.back().second.end());
+ }
+ UNIT_ASSERT_VALUES_EQUAL(items, vals);
+ }
+}
+}
+}
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
index f2b188ae47..8ee7d79e3b 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_graph.cpp
@@ -192,8 +192,8 @@ public:
private:
friend class TComputationGraphBuildingVisitor;
- friend class TComputationGraph;
-
+ friend class TComputationGraph;
+
TAllocState& AllocState;
TIntrusivePtr<TMemoryUsageInfo> MemInfo;
THolder<THolderFactory> HolderFactory;
@@ -427,8 +427,8 @@ private:
*NodeFactory,
*PatternNodes->HolderFactory,
PatternNodes->ValueBuilder.Get(),
- ValidateMode,
- ValidatePolicy,
+ ValidateMode,
+ ValidatePolicy,
GraphPerProcess,
PatternNodes->Mutables,
PatternNodes->ElementsCache,
@@ -515,9 +515,9 @@ private:
const IFunctionRegistry& FunctionRegistry;
TIntrusivePtr<TMemoryUsageInfo> MemInfo;
THolder<TNodeFactory> NodeFactory;
- NUdf::EValidateMode ValidateMode;
- NUdf::EValidatePolicy ValidatePolicy;
- EGraphPerProcess GraphPerProcess;
+ NUdf::EValidateMode ValidateMode;
+ NUdf::EValidatePolicy ValidatePolicy;
+ EGraphPerProcess GraphPerProcess;
TPatternNodes::TPtr PatternNodes;
const bool ExternalAlloc;
};
@@ -532,14 +532,14 @@ public:
#ifndef NDEBUG
CompOpts.AllocState.ActiveMemInfo.emplace(MemInfo.Get(), MemInfo);
#endif
- HolderFactory = MakeHolder<THolderFactory>(CompOpts.AllocState, *MemInfo, patternNodes->HolderFactory->GetFunctionRegistry());
+ HolderFactory = MakeHolder<THolderFactory>(CompOpts.AllocState, *MemInfo, patternNodes->HolderFactory->GetFunctionRegistry());
ValueBuilder = MakeHolder<TDefaultValueBuilder>(*HolderFactory.Get(), compOpts.ValidatePolicy);
ValueBuilder->SetSecureParamsProvider(CompOpts.SecureParamsProvider);
ArrowMemoryPool = MakeArrowMemoryPool(CompOpts.AllocState);
}
~TComputationGraph() {
- auto stats = CompOpts.Stats;
+ auto stats = CompOpts.Stats;
auto& pagePool = HolderFactory->GetPagePool();
MKQL_SET_MAX_STAT(stats, PagePool_PeakAllocated, pagePool.GetPeakAllocated());
MKQL_SET_MAX_STAT(stats, PagePool_PeakUsed, pagePool.GetPeakUsed());
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp
index 67a7cfb302..5d1a4e6b0a 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.cpp
@@ -1,6 +1,6 @@
#include "mkql_computation_node_holders.h"
#include "mkql_computation_node_codegen.h"
-#include "mkql_computation_node_pack.h"
+#include "mkql_computation_node_pack.h"
#include "mkql_custom_list.h"
#include "mkql_value_builder.h"
#include "presort.h"
@@ -1117,9 +1117,9 @@ private:
Y_VERIFY_DEBUG(IsSortedUnique());
IsBuilt = true;
- if (!Items.empty()) {
+ if (!Items.empty()) {
MKQL_MEM_TAKE(GetMemInfo(), &Items, Items.capacity() * sizeof(TItems::value_type));
- }
+ }
}
bool IsSortedUnique() const {
@@ -1196,8 +1196,8 @@ public:
private:
const NUdf::TRefCountedPtr<THashedSetHolder> Parent;
- TValuesDictHashSet::const_iterator Iterator;
- TValuesDictHashSet::const_iterator End;
+ TValuesDictHashSet::const_iterator Iterator;
+ TValuesDictHashSet::const_iterator End;
bool AtStart;
};
@@ -1269,7 +1269,7 @@ private:
bool HasDictItems() const override {
LazyBuildDict();
- return !Set.empty();
+ return !Set.empty();
}
bool IsSortedDict() const override {
@@ -1308,10 +1308,10 @@ public:
, Parent(const_cast<THashedSingleFixedSetHolder*>(parent))
, Iterator(Parent->Set.begin())
, End(Parent->Set.end())
- , AtStart(true)
- {
- }
-
+ , AtStart(true)
+ {
+ }
+
private:
bool Skip() override {
if (AtStart) {
@@ -1343,15 +1343,15 @@ public:
const NUdf::TRefCountedPtr<THashedSingleFixedSetHolder> Parent;
typename TSetType::const_iterator Iterator;
typename TSetType::const_iterator End;
- bool AtStart;
- };
-
+ bool AtStart;
+ };
+
THashedSingleFixedSetHolder(TMemoryUsageInfo* memInfo, TSetType&& set)
: TComputationValue<THashedSingleFixedSetHolder>(memInfo)
- , Set(std::move(set))
- {
- }
-
+ , Set(std::move(set))
+ {
+ }
+
private:
bool Contains(const NUdf::TUnboxedValuePod& key) const override {
return Set.find(key.Get<T>()) != Set.cend();
@@ -1372,7 +1372,7 @@ private:
NUdf::TUnboxedValue GetDictIterator() const override {
return NUdf::TUnboxedValuePod(new TIterator(this));
}
-
+
NUdf::TUnboxedValue GetPayloadsIterator() const override {
return NUdf::TUnboxedValuePod(new TIterator(this));
}
@@ -1385,24 +1385,24 @@ private:
return Set.size();
}
- bool HasDictItems() const override {
- return !Set.empty();
- }
-
+ bool HasDictItems() const override {
+ return !Set.empty();
+ }
+
bool IsSortedDict() const override {
return false;
}
const TSetType Set;
-};
-
-template <typename T>
-class THashedSingleFixedCompactSetHolder : public TComputationValue<THashedSingleFixedCompactSetHolder<T>> {
-public:
+};
+
+template <typename T>
+class THashedSingleFixedCompactSetHolder : public TComputationValue<THashedSingleFixedCompactSetHolder<T>> {
+public:
using TSetType = TValuesDictHashSingleFixedCompactSet<T>;
-
+
class TIterator : public TComputationValue<TIterator> {
- public:
+ public:
TIterator(const THashedSingleFixedCompactSetHolder* parent)
: TComputationValue<TIterator>(parent->GetMemInfo())
, Parent(const_cast<THashedSingleFixedCompactSetHolder*>(parent))
@@ -1478,11 +1478,11 @@ private:
}
ui64 GetDictLength() const override {
- return Set.Size();
+ return Set.Size();
}
bool HasDictItems() const override {
- return !Set.Empty();
+ return !Set.Empty();
}
bool IsSortedDict() const override {
@@ -1584,7 +1584,7 @@ private:
}
bool HasDictItems() const override {
- return !Set.Empty();
+ return !Set.Empty();
}
bool IsSortedDict() const override {
@@ -1594,7 +1594,7 @@ private:
private:
TPagedArena Pool;
const TSetType Set;
- mutable TValuePacker KeyPacker;
+ mutable TValuePacker KeyPacker;
TComputationContext* Ctx;
};
@@ -1631,8 +1631,8 @@ public:
if (!Skip())
return false;
key = NoSwap ?
- Parent->KeyPacker.Unpack(GetSmallValue(Iterator.Get().first), Parent->Ctx->HolderFactory):
- Parent->PayloadPacker.Unpack(GetSmallValue(Iterator.Get().second), Parent->Ctx->HolderFactory);
+ Parent->KeyPacker.Unpack(GetSmallValue(Iterator.Get().first), Parent->Ctx->HolderFactory):
+ Parent->PayloadPacker.Unpack(GetSmallValue(Iterator.Get().second), Parent->Ctx->HolderFactory);
return true;
}
@@ -1640,8 +1640,8 @@ public:
if (!Next(key))
return false;
payload = NoSwap ?
- Parent->PayloadPacker.Unpack(GetSmallValue(Iterator.Get().second), Parent->Ctx->HolderFactory):
- Parent->KeyPacker.Unpack(GetSmallValue(Iterator.Get().first), Parent->Ctx->HolderFactory);
+ Parent->PayloadPacker.Unpack(GetSmallValue(Iterator.Get().second), Parent->Ctx->HolderFactory):
+ Parent->KeyPacker.Unpack(GetSmallValue(Iterator.Get().first), Parent->Ctx->HolderFactory);
return true;
}
@@ -1674,7 +1674,7 @@ private:
auto it = Map.Find(smallValue);
if (!it.Ok())
return NUdf::TUnboxedValuePod();
- return PayloadPacker.Unpack(GetSmallValue(it.Get().second), Ctx->HolderFactory).Release().MakeOptional();
+ return PayloadPacker.Unpack(GetSmallValue(it.Get().second), Ctx->HolderFactory).Release().MakeOptional();
}
NUdf::TUnboxedValue GetKeysIterator() const override {
@@ -1694,7 +1694,7 @@ private:
}
bool HasDictItems() const override {
- return !Map.Empty();
+ return !Map.Empty();
}
bool IsSortedDict() const override {
@@ -1703,8 +1703,8 @@ private:
TPagedArena Pool;
const TMapType Map;
- mutable TValuePacker KeyPacker;
- mutable TValuePacker PayloadPacker;
+ mutable TValuePacker KeyPacker;
+ mutable TValuePacker PayloadPacker;
TComputationContext* Ctx;
};
@@ -1713,17 +1713,17 @@ public:
using TMapType = TValuesDictHashCompactMultiMap;
using TMapIterator = typename TMapType::TIterator;
- class TPayloadList: public TCustomListValue {
- public:
+ class TPayloadList: public TCustomListValue {
+ public:
class TIterator : public TComputationValue<TIterator> {
- public:
+ public:
TIterator(const THashedCompactMultiMapHolder* parent, TMapIterator from)
: TComputationValue(parent->GetMemInfo())
, Parent(const_cast<THashedCompactMultiMapHolder*>(parent))
- , Iterator(from)
- {
- }
-
+ , Iterator(from)
+ {
+ }
+
private:
bool Next(NUdf::TUnboxedValue& value) override {
if (!Iterator.Ok()) {
@@ -1745,49 +1745,49 @@ public:
}
const NUdf::TRefCountedPtr<THashedCompactMultiMapHolder> Parent;
- TMapIterator Iterator;
- };
-
- TPayloadList(TMemoryUsageInfo* memInfo, const THashedCompactMultiMapHolder* parent, TMapIterator from)
- : TCustomListValue(memInfo)
+ TMapIterator Iterator;
+ };
+
+ TPayloadList(TMemoryUsageInfo* memInfo, const THashedCompactMultiMapHolder* parent, TMapIterator from)
+ : TCustomListValue(memInfo)
, Parent(const_cast<THashedCompactMultiMapHolder*>(parent))
- , From(from)
- {
- Y_ASSERT(From.Ok());
- }
-
+ , From(from)
+ {
+ Y_ASSERT(From.Ok());
+ }
+
private:
- bool HasFastListLength() const override {
- return true;
- }
-
- ui64 GetListLength() const override {
- if (!Length) {
+ bool HasFastListLength() const override {
+ return true;
+ }
+
+ ui64 GetListLength() const override {
+ if (!Length) {
Length = Parent->Map.Count(From.GetKey());
- }
-
+ }
+
return *Length;
- }
-
- bool HasListItems() const override {
- return true;
- }
-
+ }
+
+ bool HasListItems() const override {
+ return true;
+ }
+
NUdf::TUnboxedValue GetListIterator() const override {
return NUdf::TUnboxedValuePod(new TIterator(Parent.Get(), From));
}
-
+
const NUdf::TRefCountedPtr<THashedCompactMultiMapHolder> Parent;
- TMapIterator From;
- };
-
+ TMapIterator From;
+ };
+
template <bool NoSwap>
class TIterator : public TComputationValue<TIterator<NoSwap>> {
public:
TIterator(const THashedCompactMultiMapHolder* parent)
: TComputationValue<TIterator<NoSwap>>(parent->GetMemInfo())
, Parent(const_cast<THashedCompactMultiMapHolder*>(parent))
- , Iterator(parent->Map.Iterate())
+ , Iterator(parent->Map.Iterate())
{
}
@@ -1804,7 +1804,7 @@ public:
payload = Parent->KeyPacker.Unpack(GetSmallValue(Iterator.GetKey()), Parent->CompCtx.HolderFactory);
key = Parent->CompCtx.HolderFactory.Create<TPayloadList>(Parent.Get(), Iterator.MakeCurrentKeyIter());
}
- Iterator.NextKey();
+ Iterator.NextKey();
return true;
}
@@ -1816,7 +1816,7 @@ public:
key = NoSwap ?
Parent->KeyPacker.Unpack(GetSmallValue(Iterator.GetKey()), Parent->CompCtx.HolderFactory):
NUdf::TUnboxedValue(Parent->CompCtx.HolderFactory.Create<TPayloadList>(Parent.Get(), Iterator.MakeCurrentKeyIter()));
- Iterator.NextKey();
+ Iterator.NextKey();
return true;
}
@@ -1825,12 +1825,12 @@ public:
return false;
}
- Iterator.NextKey();
+ Iterator.NextKey();
return true;
}
const NUdf::TRefCountedPtr<THashedCompactMultiMapHolder> Parent;
- TMapIterator Iterator;
+ TMapIterator Iterator;
};
THashedCompactMultiMapHolder(TMemoryUsageInfo* memInfo, TMapType&& map, TPagedArena&& pool,
@@ -1874,11 +1874,11 @@ private:
}
ui64 GetDictLength() const override {
- return Map.UniqSize();
+ return Map.UniqSize();
}
bool HasDictItems() const override {
- return !Map.Empty();
+ return !Map.Empty();
}
bool IsSortedDict() const override {
@@ -1887,8 +1887,8 @@ private:
TPagedArena Pool;
const TMapType Map;
- mutable TValuePacker KeyPacker;
- mutable TValuePacker PayloadPacker;
+ mutable TValuePacker KeyPacker;
+ mutable TValuePacker PayloadPacker;
TComputationContext& CompCtx;
};
@@ -1949,8 +1949,8 @@ public:
}
const NUdf::TRefCountedPtr<THashedDictHolder> Parent;
- TValuesDictHashMap::const_iterator Iterator;
- TValuesDictHashMap::const_iterator End;
+ TValuesDictHashMap::const_iterator Iterator;
+ TValuesDictHashMap::const_iterator End;
bool AtStart;
};
@@ -2017,7 +2017,7 @@ private:
bool HasDictItems() const override {
LazyBuildDict();
- return !Map.empty();
+ return !Map.empty();
}
bool IsSortedDict() const override {
@@ -2130,7 +2130,7 @@ private:
}
bool HasDictItems() const override {
- return !Map.empty();
+ return !Map.empty();
}
bool IsSortedDict() const override {
@@ -2140,22 +2140,22 @@ private:
const TMapType Map;
};
-template <typename T>
-class THashedSingleFixedCompactMapHolder : public TComputationValue<THashedSingleFixedCompactMapHolder<T>> {
-public:
+template <typename T>
+class THashedSingleFixedCompactMapHolder : public TComputationValue<THashedSingleFixedCompactMapHolder<T>> {
+public:
using TMapType = TValuesDictHashSingleFixedCompactMap<T>;
template <bool NoSwap>
class TIterator : public TComputationValue<TIterator<NoSwap>> {
- public:
+ public:
TIterator(const THashedSingleFixedCompactMapHolder* parent)
: TComputationValue<TIterator<NoSwap>>(parent->GetMemInfo())
, Parent(const_cast<THashedSingleFixedCompactMapHolder*>(parent))
, Iterator(Parent->Map.Iterate())
- , AtStart(true)
- {
- }
-
+ , AtStart(true)
+ {
+ }
+
private:
bool Skip() override {
if (AtStart) {
@@ -2172,8 +2172,8 @@ public:
if (!Skip())
return false;
key = NoSwap ?
- NUdf::TUnboxedValue(NUdf::TUnboxedValuePod(Iterator.Get().first)):
- Parent->PayloadPacker.Unpack(GetSmallValue(Iterator.Get().second), Parent->Ctx->HolderFactory);
+ NUdf::TUnboxedValue(NUdf::TUnboxedValuePod(Iterator.Get().first)):
+ Parent->PayloadPacker.Unpack(GetSmallValue(Iterator.Get().second), Parent->Ctx->HolderFactory);
return true;
}
@@ -2181,26 +2181,26 @@ public:
if (!Next(key))
return false;
payload = NoSwap ?
- Parent->PayloadPacker.Unpack(GetSmallValue(Iterator.Get().second), Parent->Ctx->HolderFactory):
- NUdf::TUnboxedValue(NUdf::TUnboxedValuePod(Iterator.Get().first));
+ Parent->PayloadPacker.Unpack(GetSmallValue(Iterator.Get().second), Parent->Ctx->HolderFactory):
+ NUdf::TUnboxedValue(NUdf::TUnboxedValuePod(Iterator.Get().first));
return true;
}
const NUdf::TRefCountedPtr<THashedSingleFixedCompactMapHolder> Parent;
typename TMapType::TIterator Iterator;
- bool AtStart;
- };
-
+ bool AtStart;
+ };
+
THashedSingleFixedCompactMapHolder(TMemoryUsageInfo* memInfo, TMapType&& map, TPagedArena&& pool,
- TType* payloadType, TComputationContext* ctx)
+ TType* payloadType, TComputationContext* ctx)
: TComputationValue<THashedSingleFixedCompactMapHolder>(memInfo)
, Pool(std::move(pool))
- , Map(std::move(map))
+ , Map(std::move(map))
, PayloadPacker(false, payloadType)
- , Ctx(ctx)
- {
- }
-
+ , Ctx(ctx)
+ {
+ }
+
private:
bool Contains(const NUdf::TUnboxedValuePod& key) const override {
return Map.Has(key.Get<T>());
@@ -2210,7 +2210,7 @@ private:
auto it = Map.Find(key.Get<T>());
if (!it.Ok())
return NUdf::TUnboxedValuePod();
- return PayloadPacker.Unpack(GetSmallValue(it.Get().second), Ctx->HolderFactory).Release().MakeOptional();
+ return PayloadPacker.Unpack(GetSmallValue(it.Get().second), Ctx->HolderFactory).Release().MakeOptional();
}
NUdf::TUnboxedValue GetKeysIterator() const override {
@@ -2220,7 +2220,7 @@ private:
NUdf::TUnboxedValue GetDictIterator() const override {
return NUdf::TUnboxedValuePod(new TIterator<true>(this));
}
-
+
NUdf::TUnboxedValue GetPayloadsIterator() const override {
return NUdf::TUnboxedValuePod(new TIterator<false>(this));
}
@@ -2229,38 +2229,38 @@ private:
return Map.Size();
}
- bool HasDictItems() const override {
- return !Map.Empty();
- }
-
+ bool HasDictItems() const override {
+ return !Map.Empty();
+ }
+
bool IsSortedDict() const override {
return false;
}
-private:
+private:
TPagedArena Pool;
const TMapType Map;
- mutable TValuePacker PayloadPacker;
- TComputationContext* Ctx;
-};
-
-template <typename T>
-class THashedSingleFixedCompactMultiMapHolder : public TComputationValue<THashedSingleFixedCompactMultiMapHolder<T>> {
-public:
+ mutable TValuePacker PayloadPacker;
+ TComputationContext* Ctx;
+};
+
+template <typename T>
+class THashedSingleFixedCompactMultiMapHolder : public TComputationValue<THashedSingleFixedCompactMultiMapHolder<T>> {
+public:
using TMapType = TValuesDictHashSingleFixedCompactMultiMap<T>;
using TMapIterator = typename TMapType::TIterator;
-
- class TPayloadList: public TCustomListValue {
- public:
+
+ class TPayloadList: public TCustomListValue {
+ public:
class TIterator : public TComputationValue<TIterator> {
- public:
+ public:
TIterator(const THashedSingleFixedCompactMultiMapHolder* parent, TMapIterator from)
: TComputationValue<TIterator>(parent->GetMemInfo())
, Parent(const_cast<THashedSingleFixedCompactMultiMapHolder*>(parent))
- , Iterator(from)
- {
- }
-
+ , Iterator(from)
+ {
+ }
+
private:
bool Next(NUdf::TUnboxedValue& value) override {
if (!Iterator.Ok()) {
@@ -2282,51 +2282,51 @@ public:
}
const NUdf::TRefCountedPtr<THashedSingleFixedCompactMultiMapHolder> Parent;
- TMapIterator Iterator;
- };
-
- TPayloadList(TMemoryUsageInfo* memInfo, const THashedSingleFixedCompactMultiMapHolder* parent, TMapIterator from)
- : TCustomListValue(memInfo)
+ TMapIterator Iterator;
+ };
+
+ TPayloadList(TMemoryUsageInfo* memInfo, const THashedSingleFixedCompactMultiMapHolder* parent, TMapIterator from)
+ : TCustomListValue(memInfo)
, Parent(const_cast<THashedSingleFixedCompactMultiMapHolder*>(parent))
- , From(from)
- {
- Y_ASSERT(From.Ok());
- }
-
- bool HasFastListLength() const override {
- return true;
- }
-
- ui64 GetListLength() const override {
- if (!Length) {
+ , From(from)
+ {
+ Y_ASSERT(From.Ok());
+ }
+
+ bool HasFastListLength() const override {
+ return true;
+ }
+
+ ui64 GetListLength() const override {
+ if (!Length) {
Length = Parent->Map.Count(From.GetKey());
- }
-
+ }
+
return *Length;
- }
-
- bool HasListItems() const override {
- return true;
- }
-
+ }
+
+ bool HasListItems() const override {
+ return true;
+ }
+
NUdf::TUnboxedValue GetListIterator() const override {
return NUdf::TUnboxedValuePod(new TIterator(Parent.Get(), From));
}
const NUdf::TRefCountedPtr<THashedSingleFixedCompactMultiMapHolder> Parent;
- TMapIterator From;
- };
-
+ TMapIterator From;
+ };
+
template <bool NoSwap>
class TIterator : public TComputationValue<TIterator<NoSwap>> {
- public:
+ public:
TIterator(const THashedSingleFixedCompactMultiMapHolder* parent)
: TComputationValue<TIterator<NoSwap>>(parent->GetMemInfo())
, Parent(const_cast<THashedSingleFixedCompactMultiMapHolder*>(parent))
- , Iterator(parent->Map.Iterate())
- {
- }
-
+ , Iterator(parent->Map.Iterate())
+ {
+ }
+
private:
bool Next(NUdf::TUnboxedValue& key) override {
if (!Iterator.Ok()) {
@@ -2336,7 +2336,7 @@ public:
key = NoSwap ?
NUdf::TUnboxedValuePod(Iterator.GetKey()):
Parent->Ctx->HolderFactory.template Create<TPayloadList>(Parent.Get(), Iterator.MakeCurrentKeyIter());
- Iterator.NextKey();
+ Iterator.NextKey();
return true;
}
@@ -2352,7 +2352,7 @@ public:
payload = NUdf::TUnboxedValuePod(Iterator.GetKey());
key = Parent->Ctx->HolderFactory.template Create<TPayloadList>(Parent.Get(), Iterator.MakeCurrentKeyIter());
}
- Iterator.NextKey();
+ Iterator.NextKey();
return true;
}
@@ -2361,24 +2361,24 @@ public:
return false;
}
- Iterator.NextKey();
+ Iterator.NextKey();
return true;
}
const NUdf::TRefCountedPtr<THashedSingleFixedCompactMultiMapHolder> Parent;
- TMapIterator Iterator;
- };
-
+ TMapIterator Iterator;
+ };
+
THashedSingleFixedCompactMultiMapHolder(TMemoryUsageInfo* memInfo, TMapType&& map, TPagedArena&& pool,
- TType* payloadType, TComputationContext* ctx)
+ TType* payloadType, TComputationContext* ctx)
: TComputationValue<THashedSingleFixedCompactMultiMapHolder>(memInfo)
, Pool(std::move(pool))
- , Map(std::move(map))
+ , Map(std::move(map))
, PayloadPacker(false, payloadType)
- , Ctx(ctx)
- {
- }
-
+ , Ctx(ctx)
+ {
+ }
+
private:
bool Contains(const NUdf::TUnboxedValuePod& key) const override {
return Map.Has(key.Get<T>());
@@ -2398,30 +2398,30 @@ private:
NUdf::TUnboxedValue GetDictIterator() const override {
return NUdf::TUnboxedValuePod(new TIterator<true>(this));
}
-
+
NUdf::TUnboxedValue GetPayloadsIterator() const override {
return NUdf::TUnboxedValuePod(new TIterator<false>(this));
}
ui64 GetDictLength() const override {
- return Map.UniqSize();
- }
-
- bool HasDictItems() const override {
- return !Map.Empty();
+ return Map.UniqSize();
}
+ bool HasDictItems() const override {
+ return !Map.Empty();
+ }
+
bool IsSortedDict() const override {
return false;
}
-private:
+private:
TPagedArena Pool;
const TMapType Map;
- mutable TValuePacker PayloadPacker;
- TComputationContext* Ctx;
-};
-
+ mutable TValuePacker PayloadPacker;
+ TComputationContext* Ctx;
+};
+
class TDictNode: public TMutableComputationNode<TDictNode> {
typedef TMutableComputationNode<TDictNode> TBaseComputation;
public:
@@ -3187,28 +3187,28 @@ NUdf::TUnboxedValuePod THolderFactory::ToIndexDict(const NUdf::IValueBuilder* bu
template<bool IsStream>
NUdf::TUnboxedValuePod THolderFactory::Collect(NUdf::TUnboxedValuePod list) const {
const auto boxed = list.AsBoxed(); // Only for release on exit.
- if (!IsStream && list.HasFastListLength()) {
- auto size = list.GetListLength();
- NUdf::TUnboxedValue* items = nullptr;
- const auto result = CreateDirectArrayHolder(size, items);
-
- TThresher<IsStream>::DoForEachItem(list,
- [&items] (NUdf::TUnboxedValue&& item) {
- *items++ = std::move(item);
- }
- );
- return result;
- } else {
- TDefaultListRepresentation res;
-
- TThresher<IsStream>::DoForEachItem(list,
- [&res] (NUdf::TUnboxedValue&& item) {
- res = res.Append(std::move(item));
- }
- );
-
- return CreateDirectListHolder(std::move(res));
- }
+ if (!IsStream && list.HasFastListLength()) {
+ auto size = list.GetListLength();
+ NUdf::TUnboxedValue* items = nullptr;
+ const auto result = CreateDirectArrayHolder(size, items);
+
+ TThresher<IsStream>::DoForEachItem(list,
+ [&items] (NUdf::TUnboxedValue&& item) {
+ *items++ = std::move(item);
+ }
+ );
+ return result;
+ } else {
+ TDefaultListRepresentation res;
+
+ TThresher<IsStream>::DoForEachItem(list,
+ [&res] (NUdf::TUnboxedValue&& item) {
+ res = res.Append(std::move(item));
+ }
+ );
+
+ return CreateDirectListHolder(std::move(res));
+ }
}
template NUdf::TUnboxedValuePod THolderFactory::Collect<true>(NUdf::TUnboxedValuePod list) const;
@@ -3404,33 +3404,33 @@ NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSetHolder(
template <typename T>
NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSingleFixedSetHolder(
- TValuesDictHashSingleFixedSet<T>&& set) const {
+ TValuesDictHashSingleFixedSet<T>&& set) const {
return NUdf::TUnboxedValuePod(AllocateOn<THashedSingleFixedSetHolder<T>>(CurrentAllocState, &MemInfo, std::move(set)));
}
#define DEFINE_HASHED_SINGLE_FIXED_SET(xType) \
template NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSingleFixedSetHolder<xType> \
- (TValuesDictHashSingleFixedSet<xType>&& set) const;
+ (TValuesDictHashSingleFixedSet<xType>&& set) const;
KNOWN_PRIMITIVE_VALUE_TYPES(DEFINE_HASHED_SINGLE_FIXED_SET)
-#undef DEFINE_HASHED_SINGLE_FIXED_SET
+#undef DEFINE_HASHED_SINGLE_FIXED_SET
template <typename T>
NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSingleFixedCompactSetHolder(
- TValuesDictHashSingleFixedCompactSet<T>&& set) const {
+ TValuesDictHashSingleFixedCompactSet<T>&& set) const {
return NUdf::TUnboxedValuePod(AllocateOn<THashedSingleFixedCompactSetHolder<T>>(CurrentAllocState, &MemInfo, std::move(set)));
-}
-
-#define DEFINE_HASHED_SINGLE_FIXED_COMPACT_SET(xType) \
+}
+
+#define DEFINE_HASHED_SINGLE_FIXED_COMPACT_SET(xType) \
template NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSingleFixedCompactSetHolder<xType> \
- (TValuesDictHashSingleFixedCompactSet<xType>&& set) const;
-
+ (TValuesDictHashSingleFixedCompactSet<xType>&& set) const;
+
KNOWN_PRIMITIVE_VALUE_TYPES(DEFINE_HASHED_SINGLE_FIXED_COMPACT_SET)
-#undef DEFINE_HASHED_SINGLE_FIXED_COMPACT_SET
-
-template <typename T>
+#undef DEFINE_HASHED_SINGLE_FIXED_COMPACT_SET
+
+template <typename T>
NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSingleFixedMapHolder(
- TValuesDictHashSingleFixedMap<T>&& map) const {
+ TValuesDictHashSingleFixedMap<T>&& map) const {
return NUdf::TUnboxedValuePod(AllocateOn<THashedSingleFixedMapHolder<T>>(CurrentAllocState, &MemInfo, std::move(map)));
}
@@ -3451,20 +3451,20 @@ NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedCompactMultiMapHolder(
return NUdf::TUnboxedValuePod(AllocateOn<THashedCompactMultiMapHolder>(CurrentAllocState, &MemInfo, std::move(map), std::move(pool), keyType, payloadType, ctx));
}
-template <typename T>
+template <typename T>
NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSingleFixedCompactMapHolder(
TValuesDictHashSingleFixedCompactMap<T>&& map, TPagedArena&& pool, TType* payloadType,
- TComputationContext* ctx) const {
+ TComputationContext* ctx) const {
return NUdf::TUnboxedValuePod(AllocateOn<THashedSingleFixedCompactMapHolder<T>>(CurrentAllocState, &MemInfo, std::move(map), std::move(pool), payloadType, ctx));
-}
-
-template <typename T>
+}
+
+template <typename T>
NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSingleFixedCompactMultiMapHolder(
TValuesDictHashSingleFixedCompactMultiMap<T>&& map, TPagedArena&& pool, TType* payloadType,
- TComputationContext* ctx) const {
+ TComputationContext* ctx) const {
return NUdf::TUnboxedValuePod(AllocateOn<THashedSingleFixedCompactMultiMapHolder<T>>(CurrentAllocState, &MemInfo, std::move(map), std::move(pool), payloadType, ctx));
-}
-
+}
+
NUdf::IDictValueBuilder::TPtr THolderFactory::NewDict(
const NUdf::TType* dictType,
ui32 flags) const
@@ -3480,27 +3480,27 @@ NUdf::IDictValueBuilder::TPtr THolderFactory::NewDict(
#define DEFINE_HASHED_SINGLE_FIXED_MAP(xType) \
template NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSingleFixedMapHolder<xType> \
- (TValuesDictHashSingleFixedMap<xType>&& map) const;
+ (TValuesDictHashSingleFixedMap<xType>&& map) const;
KNOWN_PRIMITIVE_VALUE_TYPES(DEFINE_HASHED_SINGLE_FIXED_MAP)
-#undef DEFINE_HASHED_SINGLE_FIXED_MAP
+#undef DEFINE_HASHED_SINGLE_FIXED_MAP
-#define DEFINE_HASHED_SINGLE_FIXED_COMPACT_MAP(xType) \
+#define DEFINE_HASHED_SINGLE_FIXED_COMPACT_MAP(xType) \
template NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSingleFixedCompactMapHolder<xType> \
(TValuesDictHashSingleFixedCompactMap<xType>&& map, TPagedArena&& pool, TType* payloadType, \
- TComputationContext* ctx) const;
-
+ TComputationContext* ctx) const;
+
KNOWN_PRIMITIVE_VALUE_TYPES(DEFINE_HASHED_SINGLE_FIXED_COMPACT_MAP)
-#undef DEFINE_HASHED_SINGLE_FIXED_COMPACT_MAP
-
-#define DEFINE_HASHED_SINGLE_FIXED_COMPACT_MULTI_MAP(xType) \
+#undef DEFINE_HASHED_SINGLE_FIXED_COMPACT_MAP
+
+#define DEFINE_HASHED_SINGLE_FIXED_COMPACT_MULTI_MAP(xType) \
template NUdf::TUnboxedValuePod THolderFactory::CreateDirectHashedSingleFixedCompactMultiMapHolder<xType> \
(TValuesDictHashSingleFixedCompactMultiMap<xType>&& map, TPagedArena&& pool, TType* payloadType, \
- TComputationContext* ctx) const;
-
+ TComputationContext* ctx) const;
+
KNOWN_PRIMITIVE_VALUE_TYPES(DEFINE_HASHED_SINGLE_FIXED_COMPACT_MULTI_MAP)
-#undef DEFINE_HASHED_SINGLE_FIXED_COMPACT_MULTI_MAP
-
+#undef DEFINE_HASHED_SINGLE_FIXED_COMPACT_MULTI_MAP
+
//////////////////////////////////////////////////////////////////////////////
// TNodeFactory
//////////////////////////////////////////////////////////////////////////////
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h
index 4486d6f43a..b0ac90b628 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_holders.h
@@ -14,9 +14,9 @@
#include <util/generic/maybe.h>
#include <util/memory/pool.h>
-#include <functional>
-#include <unordered_map>
-#include <unordered_set>
+#include <functional>
+#include <unordered_map>
+#include <unordered_set>
#ifndef MKQL_DISABLE_CODEGEN
namespace llvm {
@@ -24,7 +24,7 @@ namespace llvm {
class BasicBlock;
}
#endif
-
+
namespace NKikimr {
namespace NMiniKQL {
@@ -43,7 +43,7 @@ using TKeyPayloadPair = std::pair<NUdf::TUnboxedValue, NUdf::TUnboxedValue>;
using TKeyPayloadPairVector = std::vector<TKeyPayloadPair, TMKQLAllocator<TKeyPayloadPair>>;
inline int CompareValues(NUdf::EDataSlot type,
- bool asc, bool isOptional, const NUdf::TUnboxedValuePod& lhs, const NUdf::TUnboxedValuePod& rhs) {
+ bool asc, bool isOptional, const NUdf::TUnboxedValuePod& lhs, const NUdf::TUnboxedValuePod& rhs) {
int cmp;
if (isOptional) {
if (!lhs && !rhs) {
@@ -70,7 +70,7 @@ inline int CompareValues(NUdf::EDataSlot type,
return cmp;
}
-inline int CompareKeys(const NUdf::TUnboxedValuePod& left, const NUdf::TUnboxedValuePod& right, const TKeyTypes& types, bool isTuple) {
+inline int CompareKeys(const NUdf::TUnboxedValuePod& left, const NUdf::TUnboxedValuePod& right, const TKeyTypes& types, bool isTuple) {
if (isTuple) {
if (left && right)
for (ui32 i = 0; i < types.size(); ++i) {
@@ -118,7 +118,7 @@ struct TKeyPayloadPairEqual {
bool IsTuple;
};
-struct TValueEqual {
+struct TValueEqual {
TValueEqual(const TKeyTypes& types, bool isTuple)
: Types(&types)
, IsTuple(isTuple)
@@ -203,47 +203,47 @@ using TMyHash = std::conditional_t<std::is_floating_point<T>::value, TFloatHash<
template <typename T>
using TMyEquals = std::conditional_t<std::is_floating_point<T>::value, TFloatEquals<T>, std::equal_to<T>>;
-constexpr float COMPACT_HASH_MAX_LOAD_FACTOR = 1.2f;
+constexpr float COMPACT_HASH_MAX_LOAD_FACTOR = 1.2f;
-using TValuesDictHashMap = std::unordered_map<
- NUdf::TUnboxedValue, NUdf::TUnboxedValue,
+using TValuesDictHashMap = std::unordered_map<
+ NUdf::TUnboxedValue, NUdf::TUnboxedValue,
NYql::TVaryingHash<NUdf::TUnboxedValue, TValueHasher>, TValueEqual,
- TMKQLAllocator<std::pair<const NUdf::TUnboxedValue, NUdf::TUnboxedValue>>>;
+ TMKQLAllocator<std::pair<const NUdf::TUnboxedValue, NUdf::TUnboxedValue>>>;
-using TValuesDictHashSet = std::unordered_set<
+using TValuesDictHashSet = std::unordered_set<
NUdf::TUnboxedValue, NYql::TVaryingHash<NUdf::TUnboxedValue, TValueHasher>, TValueEqual,
- TMKQLAllocator<NUdf::TUnboxedValue>>;
-
+ TMKQLAllocator<NUdf::TUnboxedValue>>;
+
template <typename T>
using TValuesDictHashSingleFixedSet = std::unordered_set<T, NYql::TVaryingHash<T, TMyHash<T>>, TMyEquals<T>, TMKQLAllocator<T>>;
-template <typename T>
+template <typename T>
using TValuesDictHashSingleFixedCompactSet = NCHash::TCompactHashSet<T, TMyHash<T>, TMyEquals<T>>;
-
-/*
- * All *Small* functions expect the 'value' content to be formed by the TValuePacker class,
- * which embeds the encoded data length at the beginning of the buffer
- */
-inline bool IsSmallValueEmbedded(ui64 value) {
- return (value & 1) != 0;
-}
-
+
+/*
+ * All *Small* functions expect the 'value' content to be formed by the TValuePacker class,
+ * which embeds the encoded data length at the beginning of the buffer
+ */
+inline bool IsSmallValueEmbedded(ui64 value) {
+ return (value & 1) != 0;
+}
+
inline TStringBuf GetSmallValue(const ui64& value) {
- if (!IsSmallValueEmbedded(value)) {
+ if (!IsSmallValueEmbedded(value)) {
// pointer
const char* ptr = (const char*)value;
ui32 length = *(const ui32*)ptr;
- return TStringBuf(ptr, length + 4);
+ return TStringBuf(ptr, length + 4);
} else {
// embedded
- ui32 length = (value & 0x0f) >> 1;
- return TStringBuf(((const char*)&value), length + 1);
+ ui32 length = (value & 0x0f) >> 1;
+ return TStringBuf(((const char*)&value), length + 1);
}
}
inline ui64 AddSmallValue(TPagedArena& pool, const TStringBuf& value) {
if (value.size() <= 8) {
- ui64 ret = 0;
+ ui64 ret = 0;
memcpy((ui8*)&ret, value.data(), value.size());
Y_VERIFY_DEBUG(IsSmallValueEmbedded(ret));
return ret;
@@ -255,9 +255,9 @@ inline ui64 AddSmallValue(TPagedArena& pool, const TStringBuf& value) {
}
}
-inline ui64 AsSmallValue(const TStringBuf& value) {
+inline ui64 AsSmallValue(const TStringBuf& value) {
if (value.size() <= 8) {
- ui64 ret = 0;
+ ui64 ret = 0;
memcpy((ui8*)&ret, value.data(), value.size());
return ret;
}
@@ -266,9 +266,9 @@ inline ui64 AsSmallValue(const TStringBuf& value) {
}
}
-struct TSmallValueEqual {
- bool operator()(ui64 lhs, ui64 rhs) const {
- return IsSmallValueEmbedded(lhs) ? lhs == rhs : GetSmallValue(lhs) == GetSmallValue(rhs);
+struct TSmallValueEqual {
+ bool operator()(ui64 lhs, ui64 rhs) const {
+ return IsSmallValueEmbedded(lhs) ? lhs == rhs : GetSmallValue(lhs) == GetSmallValue(rhs);
}
};
@@ -278,18 +278,18 @@ struct TSmallValueHash {
}
};
-using TValuesDictHashCompactSet = NCHash::TCompactHashSet<ui64, TSmallValueHash, TSmallValueEqual>;
-using TValuesDictHashCompactMap = NCHash::TCompactHash<ui64, ui64, TSmallValueHash, TSmallValueEqual>;
-using TValuesDictHashCompactMultiMap = NCHash::TCompactMultiHash<ui64, ui64, TSmallValueHash, TSmallValueEqual>;
+using TValuesDictHashCompactSet = NCHash::TCompactHashSet<ui64, TSmallValueHash, TSmallValueEqual>;
+using TValuesDictHashCompactMap = NCHash::TCompactHash<ui64, ui64, TSmallValueHash, TSmallValueEqual>;
+using TValuesDictHashCompactMultiMap = NCHash::TCompactMultiHash<ui64, ui64, TSmallValueHash, TSmallValueEqual>;
template <typename T>
using TValuesDictHashSingleFixedCompactMap = NCHash::TCompactHash<T, ui64, TMyHash<T>, TMyEquals<T>>;
-template <typename T>
+template <typename T>
using TValuesDictHashSingleFixedCompactMultiMap = NCHash::TCompactMultiHash<T, ui64, TMyHash<T>, TMyEquals<T>>;
-
-template <typename T>
+
+template <typename T>
using TValuesDictHashSingleFixedMap = std::unordered_map<T, NUdf::TUnboxedValue, NYql::TVaryingHash<T, TMyHash<T>>, TMyEquals<T>,
- TMKQLAllocator<std::pair<const T, NUdf::TUnboxedValue>>>;
+ TMKQLAllocator<std::pair<const T, NUdf::TUnboxedValue>>>;
using THashedDictFiller = std::function<void(TValuesDictHashMap&)>;
using THashedSetFiller = std::function<void(TValuesDictHashSet&)>;
@@ -430,8 +430,8 @@ public:
template <typename T>
NUdf::TUnboxedValuePod CreateDirectHashedSingleFixedCompactSetHolder(TValuesDictHashSingleFixedCompactSet<T>&& set) const;
-
- template <typename T>
+
+ template <typename T>
NUdf::TUnboxedValuePod CreateDirectHashedSingleFixedMapHolder(TValuesDictHashSingleFixedMap<T>&& map) const;
NUdf::TUnboxedValuePod CreateDirectHashedCompactSetHolder(
@@ -446,16 +446,16 @@ public:
TValuesDictHashCompactMultiMap&& map, TPagedArena&& pool, TType* keyType, TType* payloadType,
TComputationContext* ctx) const;
- template <typename T>
+ template <typename T>
NUdf::TUnboxedValuePod CreateDirectHashedSingleFixedCompactMapHolder(
TValuesDictHashSingleFixedCompactMap<T>&& map, TPagedArena&& pool, TType* payloadType,
- TComputationContext* ctx) const;
-
- template <typename T>
+ TComputationContext* ctx) const;
+
+ template <typename T>
NUdf::TUnboxedValuePod CreateDirectHashedSingleFixedCompactMultiMapHolder(
TValuesDictHashSingleFixedCompactMultiMap<T>&& map, TPagedArena&& pool, TType* payloadType,
- TComputationContext* ctx) const;
-
+ TComputationContext* ctx) const;
+
NUdf::IDictValueBuilder::TPtr NewDict(
const NUdf::TType* dictType,
ui32 flags) const;
@@ -502,7 +502,7 @@ public:
}
}
- TAlignedPagePool& GetPagePool() const {
+ TAlignedPagePool& GetPagePool() const {
return *CurrentAllocState;
}
@@ -510,18 +510,18 @@ public:
return CurrentAllocState->GetUsed();
}
- const IFunctionRegistry* GetFunctionRegistry() const {
- return FunctionRegistry;
- }
-
+ const IFunctionRegistry* GetFunctionRegistry() const {
+ return FunctionRegistry;
+ }
+
template<bool FromStreams>
NUdf::TUnboxedValuePod ExtendList(NUdf::TUnboxedValue* data, ui64 size) const;
NUdf::TUnboxedValuePod ExtendStream(NUdf::TUnboxedValue* data, ui64 size) const;
private:
- TAllocState* const CurrentAllocState;
+ TAllocState* const CurrentAllocState;
TMemoryUsageInfo& MemInfo;
- const IFunctionRegistry* const FunctionRegistry;
+ const IFunctionRegistry* const FunctionRegistry;
const NUdf::TUnboxedValue EmptyContainer;
};
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h
index 0a63ca9d34..9b20cbd4ff 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_impl.h
@@ -1,7 +1,7 @@
#pragma once
#include "mkql_computation_node.h"
-
+
#include <ydb/library/yql/minikql/mkql_alloc.h>
#include <ydb/library/yql/public/udf/udf_value.h>
@@ -90,7 +90,7 @@ protected:
NUdf::TUnboxedValue& ValueRef(TComputationContext& compCtx) const {
return compCtx.MutableValues[ValueIndex];
- }
+ }
ui32 GetIndex() const final;
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
index 625a269eaf..cea3fbcf79 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.cpp
@@ -1,7 +1,7 @@
-#include "mkql_computation_node_pack.h"
-#include "mkql_computation_node_holders.h"
+#include "mkql_computation_node_pack.h"
+#include "mkql_computation_node_holders.h"
#include "presort.h"
-
+
#include <ydb/library/yql/public/decimal/yql_decimal.h>
#include <ydb/library/yql/public/decimal/yql_decimal_serialize.h>
#include <ydb/library/yql/minikql/defs.h>
@@ -10,7 +10,7 @@
#include <library/cpp/packedtypes/zigzag.h>
#include <library/cpp/resource/resource.h>
#include <ydb/library/yql/utils/fp_bits.h>
-
+
#ifndef MKQL_DISABLE_CODEGEN
#include <llvm/IR/Constants.h>
#include <llvm/IR/DerivedTypes.h>
@@ -19,38 +19,38 @@
#include <llvm/IR/Module.h>
#endif
-#include <util/system/yassert.h>
+#include <util/system/yassert.h>
#include <util/system/sanitizers.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
+
+namespace NKikimr {
+namespace NMiniKQL {
+
#ifndef MKQL_DISABLE_CODEGEN
using namespace llvm;
#endif
-namespace NDetails {
-
-void PackUInt64(ui64 val, TBuffer& buf) {
- size_t off = buf.Size();
- buf.Advance(MAX_PACKED64_SIZE);
- buf.EraseBack(MAX_PACKED64_SIZE - Pack64(val, buf.Data() + off));
-}
-
-void PackInt64(i64 val, TBuffer& buf) {
- PackUInt64(ZigZagEncode(val), buf);
-}
-
-void PackUInt32(ui32 val, TBuffer& buf) {
- size_t off = buf.Size();
- buf.Advance(MAX_PACKED32_SIZE);
- buf.EraseBack(MAX_PACKED32_SIZE - Pack32(val, buf.Data() + off));
-}
-
-void PackInt32(i32 val, TBuffer& buf) {
- PackUInt32(ZigZagEncode(val), buf);
-}
-
+namespace NDetails {
+
+void PackUInt64(ui64 val, TBuffer& buf) {
+ size_t off = buf.Size();
+ buf.Advance(MAX_PACKED64_SIZE);
+ buf.EraseBack(MAX_PACKED64_SIZE - Pack64(val, buf.Data() + off));
+}
+
+void PackInt64(i64 val, TBuffer& buf) {
+ PackUInt64(ZigZagEncode(val), buf);
+}
+
+void PackUInt32(ui32 val, TBuffer& buf) {
+ size_t off = buf.Size();
+ buf.Advance(MAX_PACKED32_SIZE);
+ buf.EraseBack(MAX_PACKED32_SIZE - Pack32(val, buf.Data() + off));
+}
+
+void PackInt32(i32 val, TBuffer& buf) {
+ PackUInt32(ZigZagEncode(val), buf);
+}
+
void PackUInt16(ui16 val, TBuffer& buf) {
size_t off = buf.Size();
buf.Advance(MAX_PACKED32_SIZE);
@@ -61,30 +61,30 @@ void PackInt16(i16 val, TBuffer& buf) {
PackUInt16(ZigZagEncode(val), buf);
}
-ui64 UnpackUInt64(TStringBuf& buf) {
- ui64 res = 0;
+ui64 UnpackUInt64(TStringBuf& buf) {
+ ui64 res = 0;
size_t read = Unpack64(buf.data(), buf.length(), res);
- MKQL_ENSURE(read, "Bad ui64 packed data");
- buf.Skip(read);
- return res;
-}
-
-i64 UnpackInt64(TStringBuf& buf) {
- return ZigZagDecode(UnpackUInt64(buf));
-}
-
-ui32 UnpackUInt32(TStringBuf& buf) {
- ui32 res = 0;
+ MKQL_ENSURE(read, "Bad ui64 packed data");
+ buf.Skip(read);
+ return res;
+}
+
+i64 UnpackInt64(TStringBuf& buf) {
+ return ZigZagDecode(UnpackUInt64(buf));
+}
+
+ui32 UnpackUInt32(TStringBuf& buf) {
+ ui32 res = 0;
size_t read = Unpack32(buf.data(), buf.length(), res);
- MKQL_ENSURE(read, "Bad ui32 packed data");
- buf.Skip(read);
- return res;
-}
-
-i32 UnpackInt32(TStringBuf& buf) {
- return ZigZagDecode(UnpackUInt32(buf));
-}
-
+ MKQL_ENSURE(read, "Bad ui32 packed data");
+ buf.Skip(read);
+ return res;
+}
+
+i32 UnpackInt32(TStringBuf& buf) {
+ return ZigZagDecode(UnpackUInt32(buf));
+}
+
ui16 UnpackUInt16(TStringBuf& buf) {
ui32 res = 0;
size_t read = Unpack32(buf.data(), buf.length(), res);
@@ -98,22 +98,22 @@ i16 UnpackInt16(TStringBuf& buf) {
return ZigZagDecode(UnpackUInt16(buf));
}
-template <typename T>
-void PutRawData(T val, TBuffer& buf) {
- buf.Append(reinterpret_cast<const char*>(&val), sizeof(T));
-}
-
-template <typename T>
-T GetRawData(TStringBuf& buf) {
+template <typename T>
+void PutRawData(T val, TBuffer& buf) {
+ buf.Append(reinterpret_cast<const char*>(&val), sizeof(T));
+}
+
+template <typename T>
+T GetRawData(TStringBuf& buf) {
MKQL_ENSURE(sizeof(T) <= buf.size(), "Bad packed data. Buffer too small");
- T val = 0;
+ T val = 0;
memcpy(&val, buf.data(), sizeof(T));
- buf.Skip(sizeof(T));
- return val;
-}
-
-} // NDetails
-
+ buf.Skip(sizeof(T));
+ return val;
+}
+
+} // NDetails
+
namespace {
#ifndef MKQL_DISABLE_CODEGEN
TString MakeName(const TStringBuf& common, const TType* type) {
@@ -371,32 +371,32 @@ TValuePacker::TValuePacker(const TValuePacker& other)
, PackFunc(other.PackFunc)
{}
-std::pair<ui32, bool> TValuePacker::SkipEmbeddedLength(TStringBuf& buf) {
- ui32 length = 0;
- bool emptySingleOptional = false;
+std::pair<ui32, bool> TValuePacker::SkipEmbeddedLength(TStringBuf& buf) {
+ ui32 length = 0;
+ bool emptySingleOptional = false;
if (buf.size() > 8) {
length = ReadUnaligned<ui32>(buf.data());
MKQL_ENSURE(length + 4 == buf.size(), "Bad packed data. Invalid embedded size");
- buf.Skip(4);
- } else {
+ buf.Skip(4);
+ } else {
length = *buf.data();
- MKQL_ENSURE(length & 1, "Bad packed data. Invalid embedded size");
- emptySingleOptional = 0 != (length & 0x10);
- length = (length & 0x0f) >> 1;
+ MKQL_ENSURE(length & 1, "Bad packed data. Invalid embedded size");
+ emptySingleOptional = 0 != (length & 0x10);
+ length = (length & 0x0f) >> 1;
MKQL_ENSURE(length + 1 == buf.size(), "Bad packed data. Invalid embedded size");
- buf.Skip(1);
- }
- return {length, emptySingleOptional};
-}
-
+ buf.Skip(1);
+ }
+ return {length, emptySingleOptional};
+}
+
NUdf::TUnboxedValue TValuePacker::Unpack(TStringBuf buf, const THolderFactory& holderFactory) const {
auto pair = SkipEmbeddedLength(buf);
ui32 length = pair.first;
bool emptySingleOptional = pair.second;
-
- if (Properties.Test(EProps::UseOptionalMask)) {
- OptionalUsageMask.Reset(buf);
- }
+
+ if (Properties.Test(EProps::UseOptionalMask)) {
+ OptionalUsageMask.Reset(buf);
+ }
NUdf::TUnboxedValue res;
if (Properties.Test(EProps::SingleOptional) && emptySingleOptional) {
res = NUdf::TUnboxedValuePod();
@@ -413,14 +413,14 @@ NUdf::TUnboxedValue TValuePacker::Unpack(TStringBuf buf, const THolderFactory& h
}
MKQL_ENSURE(buf.empty(), "Bad packed data. Not fully data read");
- return res;
-}
-
+ return res;
+}
+
NUdf::TUnboxedValue TValuePacker::UnpackImpl(const TType* type, TStringBuf& buf, ui32 topLength,
const THolderFactory& holderFactory) const
{
- switch (type->GetKind()) {
- case TType::EKind::Void:
+ switch (type->GetKind()) {
+ case TType::EKind::Void:
return NUdf::TUnboxedValuePod::Void();
case TType::EKind::Null:
return NUdf::TUnboxedValuePod();
@@ -428,8 +428,8 @@ NUdf::TUnboxedValue TValuePacker::UnpackImpl(const TType* type, TStringBuf& buf,
return holderFactory.GetEmptyContainer();
case TType::EKind::EmptyDict:
return holderFactory.GetEmptyContainer();
-
- case TType::EKind::Data: {
+
+ case TType::EKind::Data: {
auto dataType = static_cast<const TDataType*>(type);
switch (*dataType->GetDataSlot()) {
case NUdf::EDataSlot::Bool:
@@ -495,177 +495,177 @@ NUdf::TUnboxedValue TValuePacker::UnpackImpl(const TType* type, TStringBuf& buf,
buf.Skip(des.second);
return NUdf::TUnboxedValuePod(des.first);
}
- default:
- ui32 size = 0;
- if (Properties.Test(EProps::UseTopLength)) {
- size = topLength;
- } else {
- size = NDetails::UnpackUInt32(buf);
- }
+ default:
+ ui32 size = 0;
+ if (Properties.Test(EProps::UseTopLength)) {
+ size = topLength;
+ } else {
+ size = NDetails::UnpackUInt32(buf);
+ }
MKQL_ENSURE(size <= buf.size(), "Bad packed data. Buffer too small");
const char* ptr = buf.data();
- buf.Skip(size);
+ buf.Skip(size);
return MakeString(NUdf::TStringRef(ptr, size));
- }
- break;
- }
-
- case TType::EKind::Optional: {
+ }
+ break;
+ }
+
+ case TType::EKind::Optional: {
auto optionalType = static_cast<const TOptionalType*>(type);
- if (!OptionalUsageMask.IsNextEmptyOptional()) {
+ if (!OptionalUsageMask.IsNextEmptyOptional()) {
return UnpackImpl(optionalType->GetItemType(), buf, topLength, holderFactory).Release().MakeOptional();
- }
- else {
+ }
+ else {
return NUdf::TUnboxedValuePod();
- }
- }
-
- case TType::EKind::List: {
+ }
+ }
+
+ case TType::EKind::List: {
auto listType = static_cast<const TListType*>(type);
- auto itemType = listType->GetItemType();
+ auto itemType = listType->GetItemType();
const auto len = NDetails::UnpackUInt64(buf);
NUdf::TUnboxedValue *items = nullptr;
auto list = holderFactory.CreateDirectArrayHolder(len, items);
- for (ui64 i = 0; i < len; ++i) {
+ for (ui64 i = 0; i < len; ++i) {
*items++ = UnpackImpl(itemType, buf, topLength, holderFactory);
- }
+ }
return std::move(list);
- }
-
- case TType::EKind::Struct: {
+ }
+
+ case TType::EKind::Struct: {
auto structType = static_cast<const TStructType*>(type);
- NUdf::TUnboxedValue* itemsPtr = nullptr;
+ NUdf::TUnboxedValue* itemsPtr = nullptr;
auto res = holderFactory.CreateDirectArrayHolder(structType->GetMembersCount(), itemsPtr);
- for (ui32 index = 0; index < structType->GetMembersCount(); ++index) {
- auto memberType = structType->GetMemberType(index);
+ for (ui32 index = 0; index < structType->GetMembersCount(); ++index) {
+ auto memberType = structType->GetMemberType(index);
itemsPtr[index] = UnpackImpl(memberType, buf, topLength, holderFactory);
- }
+ }
return std::move(res);
- }
-
- case TType::EKind::Tuple: {
+ }
+
+ case TType::EKind::Tuple: {
auto tupleType = static_cast<const TTupleType*>(type);
- NUdf::TUnboxedValue* itemsPtr = nullptr;
+ NUdf::TUnboxedValue* itemsPtr = nullptr;
auto res = holderFactory.CreateDirectArrayHolder(tupleType->GetElementsCount(), itemsPtr);
- for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) {
- auto elementType = tupleType->GetElementType(index);
+ for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) {
+ auto elementType = tupleType->GetElementType(index);
itemsPtr[index] = UnpackImpl(elementType, buf, topLength, holderFactory);
- }
+ }
return std::move(res);
- }
-
- case TType::EKind::Dict: {
+ }
+
+ case TType::EKind::Dict: {
auto dictType = static_cast<const TDictType*>(type);
- auto keyType = dictType->GetKeyType();
- auto payloadType = dictType->GetPayloadType();
+ auto keyType = dictType->GetKeyType();
+ auto payloadType = dictType->GetPayloadType();
auto dictBuilder = holderFactory.NewDict(dictType, NUdf::TDictFlags::EDictKind::Hashed);
-
- ui64 len = NDetails::UnpackUInt64(buf);
- for (ui64 i = 0; i < len; ++i) {
+
+ ui64 len = NDetails::UnpackUInt64(buf);
+ for (ui64 i = 0; i < len; ++i) {
auto key = UnpackImpl(keyType, buf, topLength, holderFactory);
auto payload = UnpackImpl(payloadType, buf, topLength, holderFactory);
dictBuilder->Add(std::move(key), std::move(payload));
- }
- return dictBuilder->Build();
- }
-
- case TType::EKind::Variant: {
+ }
+ return dictBuilder->Build();
+ }
+
+ case TType::EKind::Variant: {
auto variantType = static_cast<const TVariantType*>(type);
- ui32 variantIndex = NDetails::UnpackUInt32(buf);
- TType* innerType = variantType->GetUnderlyingType();
- if (innerType->IsStruct()) {
- innerType = static_cast<TStructType*>(innerType)->GetMemberType(variantIndex);
- } else {
- MKQL_ENSURE(innerType->IsTuple(), "Unexpected underlying variant type: " << innerType->GetKindAsStr());
- innerType = static_cast<TTupleType*>(innerType)->GetElementType(variantIndex);
- }
+ ui32 variantIndex = NDetails::UnpackUInt32(buf);
+ TType* innerType = variantType->GetUnderlyingType();
+ if (innerType->IsStruct()) {
+ innerType = static_cast<TStructType*>(innerType)->GetMemberType(variantIndex);
+ } else {
+ MKQL_ENSURE(innerType->IsTuple(), "Unexpected underlying variant type: " << innerType->GetKindAsStr());
+ innerType = static_cast<TTupleType*>(innerType)->GetElementType(variantIndex);
+ }
return holderFactory.CreateVariantHolder(UnpackImpl(innerType, buf, topLength, holderFactory).Release(), variantIndex);
- }
-
+ }
+
case TType::EKind::Tagged: {
auto taggedType = static_cast<const TTaggedType*>(type);
return UnpackImpl(taggedType->GetBaseType(), buf, topLength, holderFactory);
}
- default:
- THROW yexception() << "Unsupported type: " << type->GetKindAsStr();
- }
-}
-
+ default:
+ THROW yexception() << "Unsupported type: " << type->GetKindAsStr();
+ }
+}
+
TStringBuf TValuePacker::Pack(const NUdf::TUnboxedValuePod& value) const {
- OptionalUsageMask.Reset();
+ OptionalUsageMask.Reset();
const size_t lengthReserve = sizeof(ui32);
- Buffer.Proceed(lengthReserve + OptionalMaskReserve);
-
+ Buffer.Proceed(lengthReserve + OptionalMaskReserve);
+
if (PackFunc)
PackFunc(reinterpret_cast<const TRawUV*>(&value), reinterpret_cast<ui64*>(&Buffer), reinterpret_cast<ui64*>(&OptionalUsageMask));
else
PackImpl(Type, value);
- size_t delta = 0;
- size_t len = Buffer.Size();
-
- if (Properties.Test(EProps::UseOptionalMask)) {
- // Prepend optional mask
+ size_t delta = 0;
+ size_t len = Buffer.Size();
+
+ if (Properties.Test(EProps::UseOptionalMask)) {
+ // Prepend optional mask
const size_t actualOptionalMaskSize = OptionalUsageMask.CalcSerializedSize();
-
- if (actualOptionalMaskSize > OptionalMaskReserve) {
- TBuffer buf(Buffer.Size() + actualOptionalMaskSize - OptionalMaskReserve);
- buf.Proceed(actualOptionalMaskSize - OptionalMaskReserve);
- buf.Append(Buffer.Data(), Buffer.Size());
- Buffer.Swap(buf);
- OptionalMaskReserve = actualOptionalMaskSize;
- len = Buffer.Size();
- }
-
- delta = OptionalMaskReserve - actualOptionalMaskSize;
- Buffer.Proceed(lengthReserve + delta);
- OptionalUsageMask.Serialize(Buffer);
- }
-
- // Prepend length
- if (len - delta - lengthReserve > 7) {
+
+ if (actualOptionalMaskSize > OptionalMaskReserve) {
+ TBuffer buf(Buffer.Size() + actualOptionalMaskSize - OptionalMaskReserve);
+ buf.Proceed(actualOptionalMaskSize - OptionalMaskReserve);
+ buf.Append(Buffer.Data(), Buffer.Size());
+ Buffer.Swap(buf);
+ OptionalMaskReserve = actualOptionalMaskSize;
+ len = Buffer.Size();
+ }
+
+ delta = OptionalMaskReserve - actualOptionalMaskSize;
+ Buffer.Proceed(lengthReserve + delta);
+ OptionalUsageMask.Serialize(Buffer);
+ }
+
+ // Prepend length
+ if (len - delta - lengthReserve > 7) {
const ui32 length = len - delta - lengthReserve;
- Buffer.Proceed(delta);
- Buffer.Append((const char*)&length, sizeof(length));
- // Long length always singnals non-empty optional. So, don't check EProps::SingleOptional here
- } else {
- ui8 length = 1 | ((len - delta - lengthReserve) << 1);
- // Empty root optional always has short length. Embed empty flag into the length
- if (Properties.Test(EProps::SingleOptional) && !OptionalUsageMask.IsEmptyMask()) {
- length |= 0x10;
- }
- delta += 3;
- Buffer.Proceed(delta);
- Buffer.Append((const char*)&length, sizeof(length));
- }
+ Buffer.Proceed(delta);
+ Buffer.Append((const char*)&length, sizeof(length));
+ // Long length always singnals non-empty optional. So, don't check EProps::SingleOptional here
+ } else {
+ ui8 length = 1 | ((len - delta - lengthReserve) << 1);
+ // Empty root optional always has short length. Embed empty flag into the length
+ if (Properties.Test(EProps::SingleOptional) && !OptionalUsageMask.IsEmptyMask()) {
+ length |= 0x10;
+ }
+ delta += 3;
+ Buffer.Proceed(delta);
+ Buffer.Append((const char*)&length, sizeof(length));
+ }
NSan::Unpoison(Buffer.Data() + delta, len - delta);
- return TStringBuf(Buffer.Data() + delta, len - delta);
-}
-
+ return TStringBuf(Buffer.Data() + delta, len - delta);
+}
+
void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& value) const {
- switch (type->GetKind()) {
- case TType::EKind::Void:
- break;
+ switch (type->GetKind()) {
+ case TType::EKind::Void:
+ break;
case TType::EKind::Null:
break;
case TType::EKind::EmptyList:
break;
case TType::EKind::EmptyDict:
break;
-
- case TType::EKind::Data: {
+
+ case TType::EKind::Data: {
auto dataType = static_cast<const TDataType*>(type);
switch (*dataType->GetDataSlot()) {
case NUdf::EDataSlot::Bool:
- NDetails::PutRawData(value.Get<bool>(), Buffer);
- break;
+ NDetails::PutRawData(value.Get<bool>(), Buffer);
+ break;
case NUdf::EDataSlot::Int8:
NDetails::PutRawData(value.Get<i8>(), Buffer);
break;
case NUdf::EDataSlot::Uint8:
- NDetails::PutRawData(value.Get<ui8>(), Buffer);
- break;
+ NDetails::PutRawData(value.Get<ui8>(), Buffer);
+ break;
case NUdf::EDataSlot::Int16:
NDetails::PackInt16(value.Get<i16>(), Buffer);
break;
@@ -673,17 +673,17 @@ void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& val
NDetails::PackUInt16(value.Get<ui16>(), Buffer);
break;
case NUdf::EDataSlot::Int32:
- NDetails::PackInt32(value.Get<i32>(), Buffer);
- break;
+ NDetails::PackInt32(value.Get<i32>(), Buffer);
+ break;
case NUdf::EDataSlot::Uint32:
- NDetails::PackUInt32(value.Get<ui32>(), Buffer);
- break;
+ NDetails::PackUInt32(value.Get<ui32>(), Buffer);
+ break;
case NUdf::EDataSlot::Int64:
- NDetails::PackInt64(value.Get<i64>(), Buffer);
- break;
+ NDetails::PackInt64(value.Get<i64>(), Buffer);
+ break;
case NUdf::EDataSlot::Uint64:
- NDetails::PackUInt64(value.Get<ui64>(), Buffer);
- break;
+ NDetails::PackUInt64(value.Get<ui64>(), Buffer);
+ break;
case NUdf::EDataSlot::Float: {
float x = value.Get<float>();
if (Stable) {
@@ -691,7 +691,7 @@ void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& val
}
NDetails::PutRawData(x, Buffer);
- break;
+ break;
}
case NUdf::EDataSlot::Double: {
double x = value.Get<double>();
@@ -700,7 +700,7 @@ void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& val
}
NDetails::PutRawData(x, Buffer);
- break;
+ break;
}
case NUdf::EDataSlot::Date:
NDetails::PackUInt32(value.Get<ui16>(), Buffer);
@@ -740,28 +740,28 @@ void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& val
break;
}
default: {
- auto stringRef = value.AsStringRef();
- if (!Properties.Test(EProps::UseTopLength)) {
- NDetails::PackUInt32(stringRef.Size(), Buffer);
- }
- Buffer.Append(stringRef.Data(), stringRef.Size());
- }
- }
- break;
- }
-
- case TType::EKind::Optional: {
+ auto stringRef = value.AsStringRef();
+ if (!Properties.Test(EProps::UseTopLength)) {
+ NDetails::PackUInt32(stringRef.Size(), Buffer);
+ }
+ Buffer.Append(stringRef.Data(), stringRef.Size());
+ }
+ }
+ break;
+ }
+
+ case TType::EKind::Optional: {
auto optionalType = static_cast<const TOptionalType*>(type);
OptionalUsageMask.SetNextEmptyOptional(!value);
if (value) {
- PackImpl(optionalType->GetItemType(), value.GetOptionalValue());
- }
- break;
- }
-
- case TType::EKind::List: {
+ PackImpl(optionalType->GetItemType(), value.GetOptionalValue());
+ }
+ break;
+ }
+
+ case TType::EKind::List: {
auto listType = static_cast<const TListType*>(type);
- auto itemType = listType->GetItemType();
+ auto itemType = listType->GetItemType();
if (value.HasFastListLength()) {
auto len = value.GetListLength();
NDetails::PackUInt64(len, Buffer);
@@ -786,33 +786,33 @@ void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& val
for (const auto& item : items) {
PackImpl(itemType, item);
}
- }
- break;
- }
-
- case TType::EKind::Struct: {
+ }
+ break;
+ }
+
+ case TType::EKind::Struct: {
auto structType = static_cast<const TStructType*>(type);
- for (ui32 index = 0; index < structType->GetMembersCount(); ++index) {
- auto memberType = structType->GetMemberType(index);
+ for (ui32 index = 0; index < structType->GetMembersCount(); ++index) {
+ auto memberType = structType->GetMemberType(index);
PackImpl(memberType, value.GetElement(index));
- }
- break;
- }
-
- case TType::EKind::Tuple: {
+ }
+ break;
+ }
+
+ case TType::EKind::Tuple: {
auto tupleType = static_cast<const TTupleType*>(type);
- for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) {
- auto elementType = tupleType->GetElementType(index);
+ for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) {
+ auto elementType = tupleType->GetElementType(index);
PackImpl(elementType, value.GetElement(index));
- }
- break;
- }
-
- case TType::EKind::Dict: {
+ }
+ break;
+ }
+
+ case TType::EKind::Dict: {
auto dictType = static_cast<const TDictType*>(type);
- auto keyType = dictType->GetKeyType();
- auto payloadType = dictType->GetPayloadType();
-
+ auto keyType = dictType->GetKeyType();
+ auto payloadType = dictType->GetPayloadType();
+
auto length = value.GetDictLength();
NDetails::PackUInt64(length, Buffer);
const auto iter = value.GetDictIterator();
@@ -824,166 +824,166 @@ void TValuePacker::PackImpl(const TType* type, const NUdf::TUnboxedValuePod& val
GetDictionaryKeyTypes(keyType, types, isTuple, encoded);
if (encoded) {
TGenericPresortEncoder packer(keyType);
- decltype(EncodedDictBuffers)::value_type dictBuffer;
- if (!EncodedDictBuffers.empty()) {
- dictBuffer = std::move(EncodedDictBuffers.back());
- EncodedDictBuffers.pop_back();
- dictBuffer.clear();
- }
- dictBuffer.reserve(length);
+ decltype(EncodedDictBuffers)::value_type dictBuffer;
+ if (!EncodedDictBuffers.empty()) {
+ dictBuffer = std::move(EncodedDictBuffers.back());
+ EncodedDictBuffers.pop_back();
+ dictBuffer.clear();
+ }
+ dictBuffer.reserve(length);
for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload);) {
NUdf::TUnboxedValue encodedKey = MakeString(packer.Encode(key, false));
- dictBuffer.emplace_back(std::move(encodedKey), std::move(key), std::move(payload));
+ dictBuffer.emplace_back(std::move(encodedKey), std::move(key), std::move(payload));
}
- Sort(dictBuffer.begin(), dictBuffer.end(), [&](const auto& left, const auto& right) {
+ Sort(dictBuffer.begin(), dictBuffer.end(), [&](const auto& left, const auto& right) {
return CompareKeys(std::get<0>(left), std::get<0>(right), types, isTuple) < 0;
});
- for (const auto& x : dictBuffer) {
+ for (const auto& x : dictBuffer) {
PackImpl(keyType, std::get<1>(x));
PackImpl(payloadType, std::get<2>(x));
}
- dictBuffer.clear();
- EncodedDictBuffers.push_back(std::move(dictBuffer));
+ dictBuffer.clear();
+ EncodedDictBuffers.push_back(std::move(dictBuffer));
} else {
- decltype(DictBuffers)::value_type dictBuffer;
- if (!DictBuffers.empty()) {
- dictBuffer = std::move(DictBuffers.back());
- DictBuffers.pop_back();
- dictBuffer.clear();
- }
- dictBuffer.reserve(length);
+ decltype(DictBuffers)::value_type dictBuffer;
+ if (!DictBuffers.empty()) {
+ dictBuffer = std::move(DictBuffers.back());
+ DictBuffers.pop_back();
+ dictBuffer.clear();
+ }
+ dictBuffer.reserve(length);
for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload);) {
- dictBuffer.emplace_back(std::move(key), std::move(payload));
+ dictBuffer.emplace_back(std::move(key), std::move(payload));
}
Sort(dictBuffer.begin(), dictBuffer.end(), TKeyPayloadPairLess(types, isTuple));
- for (const auto& p: dictBuffer) {
- PackImpl(keyType, p.first);
- PackImpl(payloadType, p.second);
+ for (const auto& p: dictBuffer) {
+ PackImpl(keyType, p.first);
+ PackImpl(payloadType, p.second);
}
- dictBuffer.clear();
- DictBuffers.push_back(std::move(dictBuffer));
+ dictBuffer.clear();
+ DictBuffers.push_back(std::move(dictBuffer));
}
} else {
for (NUdf::TUnboxedValue key, payload; iter.NextPair(key, payload);) {
PackImpl(keyType, key);
PackImpl(payloadType, payload);
}
- }
- break;
- }
-
- case TType::EKind::Variant: {
+ }
+ break;
+ }
+
+ case TType::EKind::Variant: {
auto variantType = static_cast<const TVariantType*>(type);
- ui32 variantIndex = value.GetVariantIndex();
- TType* innerType = variantType->GetUnderlyingType();
- if (innerType->IsStruct()) {
- innerType = static_cast<TStructType*>(innerType)->GetMemberType(variantIndex);
- } else {
- MKQL_ENSURE(innerType->IsTuple(), "Unexpected underlying variant type: " << innerType->GetKindAsStr());
- innerType = static_cast<TTupleType*>(innerType)->GetElementType(variantIndex);
- }
- NDetails::PackUInt32(variantIndex, Buffer);
- PackImpl(innerType, value.GetVariantItem());
- break;
- }
-
- default:
- THROW yexception() << "Unsupported type: " << type->GetKindAsStr();
- }
-}
-
+ ui32 variantIndex = value.GetVariantIndex();
+ TType* innerType = variantType->GetUnderlyingType();
+ if (innerType->IsStruct()) {
+ innerType = static_cast<TStructType*>(innerType)->GetMemberType(variantIndex);
+ } else {
+ MKQL_ENSURE(innerType->IsTuple(), "Unexpected underlying variant type: " << innerType->GetKindAsStr());
+ innerType = static_cast<TTupleType*>(innerType)->GetElementType(variantIndex);
+ }
+ NDetails::PackUInt32(variantIndex, Buffer);
+ PackImpl(innerType, value.GetVariantItem());
+ break;
+ }
+
+ default:
+ THROW yexception() << "Unsupported type: " << type->GetKindAsStr();
+ }
+}
+
TValuePacker::TProperties TValuePacker::ScanTypeProperties(const TType* type) {
- TProperties props;
- if (HasOptionalFields(type)) {
- props.Set(EProps::UseOptionalMask);
- }
- if (type->GetKind() == TType::EKind::Optional) {
+ TProperties props;
+ if (HasOptionalFields(type)) {
+ props.Set(EProps::UseOptionalMask);
+ }
+ if (type->GetKind() == TType::EKind::Optional) {
type = static_cast<const TOptionalType*>(type)->GetItemType();
- if (!HasOptionalFields(type)) {
- props.Set(EProps::SingleOptional);
- props.Reset(EProps::UseOptionalMask);
- }
- }
- // Here and after the type is unwrapped!!
-
- if (type->GetKind() == TType::EKind::Data) {
+ if (!HasOptionalFields(type)) {
+ props.Set(EProps::SingleOptional);
+ props.Reset(EProps::UseOptionalMask);
+ }
+ }
+ // Here and after the type is unwrapped!!
+
+ if (type->GetKind() == TType::EKind::Data) {
auto dataType = static_cast<const TDataType*>(type);
switch (*dataType->GetDataSlot()) {
- case NUdf::EDataSlot::String:
- case NUdf::EDataSlot::Json:
- case NUdf::EDataSlot::Yson:
- case NUdf::EDataSlot::Utf8:
+ case NUdf::EDataSlot::String:
+ case NUdf::EDataSlot::Json:
+ case NUdf::EDataSlot::Yson:
+ case NUdf::EDataSlot::Utf8:
case NUdf::EDataSlot::JsonDocument:
// Reuse entire packed value length for strings
- props.Set(EProps::UseTopLength);
- break;
- default:
- break;
- }
- }
- return props;
-}
-
+ props.Set(EProps::UseTopLength);
+ break;
+ default:
+ break;
+ }
+ }
+ return props;
+}
+
bool TValuePacker::HasOptionalFields(const TType* type) {
- switch (type->GetKind()) {
- case TType::EKind::Void:
+ switch (type->GetKind()) {
+ case TType::EKind::Void:
case TType::EKind::Null:
case TType::EKind::EmptyList:
case TType::EKind::EmptyDict:
- case TType::EKind::Data:
- return false;
-
- case TType::EKind::Optional:
- return true;
-
- case TType::EKind::List:
+ case TType::EKind::Data:
+ return false;
+
+ case TType::EKind::Optional:
+ return true;
+
+ case TType::EKind::List:
return HasOptionalFields(static_cast<const TListType*>(type)->GetItemType());
-
- case TType::EKind::Struct: {
+
+ case TType::EKind::Struct: {
auto structType = static_cast<const TStructType*>(type);
- for (ui32 index = 0; index < structType->GetMembersCount(); ++index) {
- if (HasOptionalFields(structType->GetMemberType(index))) {
- return true;
- }
- }
- return false;
- }
-
- case TType::EKind::Tuple: {
+ for (ui32 index = 0; index < structType->GetMembersCount(); ++index) {
+ if (HasOptionalFields(structType->GetMemberType(index))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ case TType::EKind::Tuple: {
auto tupleType = static_cast<const TTupleType*>(type);
- for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) {
- if (HasOptionalFields(tupleType->GetElementType(index))) {
- return true;
- }
- }
- return false;
- }
-
- case TType::EKind::Dict: {
+ for (ui32 index = 0; index < tupleType->GetElementsCount(); ++index) {
+ if (HasOptionalFields(tupleType->GetElementType(index))) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ case TType::EKind::Dict: {
auto dictType = static_cast<const TDictType*>(type);
- return HasOptionalFields(dictType->GetKeyType()) || HasOptionalFields(dictType->GetPayloadType());
- }
-
- case TType::EKind::Variant: {
+ return HasOptionalFields(dictType->GetKeyType()) || HasOptionalFields(dictType->GetPayloadType());
+ }
+
+ case TType::EKind::Variant: {
auto variantType = static_cast<const TVariantType*>(type);
- return HasOptionalFields(variantType->GetUnderlyingType());
- }
-
+ return HasOptionalFields(variantType->GetUnderlyingType());
+ }
+
case TType::EKind::Tagged: {
auto taggedType = static_cast<const TTaggedType*>(type);
return HasOptionalFields(taggedType->GetBaseType());
}
- default:
- THROW yexception() << "Unsupported type: " << type->GetKindAsStr();
- }
-}
-
+ default:
+ THROW yexception() << "Unsupported type: " << type->GetKindAsStr();
+ }
+}
+
TValuePacker::TPackFunction
TValuePacker::MakePackFunction() {
#ifdef MKQL_DISABLE_CODEGEN
@@ -1007,5 +1007,5 @@ TValuePackerBoxed::TValuePackerBoxed(TMemoryUsageInfo* memInfo, const TValuePack
, TValuePacker(other)
{}
-} // NMiniKQL
-} // NKikimr
+} // NMiniKQL
+} // NKikimr
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h
index c69f4467a3..5c0020d3db 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack.h
@@ -1,54 +1,54 @@
-#pragma once
-
-#include "mkql_computation_node.h"
+#pragma once
+
+#include "mkql_computation_node.h"
#include "mkql_computation_node_holders.h"
#include "mkql_optional_usage_mask.h"
-
+
#include <ydb/library/yql/public/udf/udf_value.h>
-
+
#include <library/cpp/enumbitset/enumbitset.h>
-
+
#include <util/stream/output.h>
-#include <util/generic/buffer.h>
-#include <util/generic/strbuf.h>
-
+#include <util/generic/buffer.h>
+#include <util/generic/strbuf.h>
+
#ifndef MKQL_DISABLE_CODEGEN
#include <ydb/library/yql/minikql/codegen/codegen.h>
#endif
-#include <utility>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
-class TValuePacker {
-private:
- enum EProps {
- Begin,
- UseOptionalMask = Begin,
- UseTopLength,
- SingleOptional,
- End
- };
- using TProperties = TEnumBitSet<EProps, EProps::Begin, EProps::End>;
-public:
+#include <utility>
+
+namespace NKikimr {
+namespace NMiniKQL {
+
+class TValuePacker {
+private:
+ enum EProps {
+ Begin,
+ UseOptionalMask = Begin,
+ UseTopLength,
+ SingleOptional,
+ End
+ };
+ using TProperties = TEnumBitSet<EProps, EProps::Begin, EProps::End>;
+public:
TValuePacker(bool stable, const TType* type, bool tryUseCodegen = false);
TValuePacker(const TValuePacker& other);
-
- // Returned buffer is temporary and should be copied before next Pack() call
+
+ // Returned buffer is temporary and should be copied before next Pack() call
TStringBuf Pack(const NUdf::TUnboxedValuePod& value) const;
NUdf::TUnboxedValue Unpack(TStringBuf buf, const THolderFactory& holderFactory) const;
-
-private:
+
+private:
void PackImpl(const TType* type, const NUdf::TUnboxedValuePod& value) const;
NUdf::TUnboxedValue UnpackImpl(const TType* type, TStringBuf& buf, ui32 topLength, const THolderFactory& holderFactory) const;
static TProperties ScanTypeProperties(const TType* type);
static bool HasOptionalFields(const TType* type);
- // Returns length and empty single optional flag
- static std::pair<ui32, bool> SkipEmbeddedLength(TStringBuf& buf);
+ // Returns length and empty single optional flag
+ static std::pair<ui32, bool> SkipEmbeddedLength(TStringBuf& buf);
typedef void(*TPackFunction)(const TRawUV*, ui64*, ui64*);
TPackFunction MakePackFunction();
-
+
#ifndef MKQL_DISABLE_CODEGEN
const NYql::NCodegen::ICodegen::TPtr Codegen;
#endif
@@ -56,17 +56,17 @@ private:
const TType* Type;
// TODO: real thread safety with external state
mutable TBuffer Buffer;
- TProperties Properties;
+ TProperties Properties;
mutable size_t OptionalMaskReserve;
mutable NDetails::TOptionalUsageMask OptionalUsageMask;
mutable TPlainContainerCache TopStruct;
- mutable TVector<TVector<std::pair<NUdf::TUnboxedValue, NUdf::TUnboxedValue>>> DictBuffers;
- mutable TVector<TVector<std::tuple<NUdf::TUnboxedValue, NUdf::TUnboxedValue, NUdf::TUnboxedValue>>> EncodedDictBuffers;
+ mutable TVector<TVector<std::pair<NUdf::TUnboxedValue, NUdf::TUnboxedValue>>> DictBuffers;
+ mutable TVector<TVector<std::tuple<NUdf::TUnboxedValue, NUdf::TUnboxedValue, NUdf::TUnboxedValue>>> EncodedDictBuffers;
TPackFunction PackFunc = nullptr;
friend struct TValuePackerDetails;
-};
-
+};
+
class TValuePackerBoxed : public TComputationValue<TValuePackerBoxed>, public TValuePacker {
typedef TComputationValue<TValuePackerBoxed> TBase;
public:
@@ -74,5 +74,5 @@ public:
TValuePackerBoxed(TMemoryUsageInfo* memInfo, const TValuePacker& other);
};
-}
-}
+}
+}
diff --git a/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp b/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp
index 6a47b51829..46406c2f1e 100644
--- a/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_computation_node_pack_ut.cpp
@@ -1,28 +1,28 @@
-#include "mkql_computation_node_pack.h"
-#include "mkql_computation_node_holders.h"
+#include "mkql_computation_node_pack.h"
+#include "mkql_computation_node_holders.h"
#include <ydb/library/yql/minikql/mkql_function_registry.h>
#include <ydb/library/yql/minikql/mkql_node.h>
#include <ydb/library/yql/minikql/mkql_program_builder.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
-
+
#include <library/cpp/random_provider/random_provider.h>
#include <ydb/library/yql/minikql/aligned_page_pool.h>
-
+
#include <ydb/library/yql/public/udf/udf_value.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-#include <util/generic/ylimits.h>
-#include <util/generic/xrange.h>
-#include <util/generic/maybe.h>
-#include <util/string/cast.h>
-#include <util/string/builder.h>
+#include <util/generic/ylimits.h>
+#include <util/generic/xrange.h>
+#include <util/generic/maybe.h>
+#include <util/string/cast.h>
+#include <util/string/builder.h>
#include <util/system/hp_timer.h>
#include <util/system/sanitizers.h>
-
-namespace NKikimr {
-namespace NMiniKQL {
-
+
+namespace NKikimr {
+namespace NMiniKQL {
+
#ifdef WITH_VALGRIND
constexpr static size_t PERFORMANCE_COUNT = 0x1000;
#else
@@ -30,19 +30,19 @@ constexpr static size_t PERFORMANCE_COUNT = NSan::PlainOrUnderSanitizer(0x400000
#endif
template<bool UseCodegen>
-class TMiniKQLComputationNodePackTest: public TTestBase {
+class TMiniKQLComputationNodePackTest: public TTestBase {
protected:
- TMiniKQLComputationNodePackTest()
+ TMiniKQLComputationNodePackTest()
: FunctionRegistry(CreateFunctionRegistry(CreateBuiltinRegistry()))
- , RandomProvider(CreateDefaultRandomProvider())
+ , RandomProvider(CreateDefaultRandomProvider())
, Env(Alloc)
, PgmBuilder(Env, *FunctionRegistry)
, MemInfo("Memory")
, HolderFactory(Alloc.Ref(), MemInfo, FunctionRegistry.Get())
- {
- }
-
- void TestNumericTypes() {
+ {
+ }
+
+ void TestNumericTypes() {
TestNumericType<bool>(NUdf::TDataType<bool>::Id);
TestNumericType<ui8>(NUdf::TDataType<ui8>::Id);
TestNumericType<i32>(NUdf::TDataType<i32>::Id);
@@ -51,9 +51,9 @@ protected:
TestNumericType<ui64>(NUdf::TDataType<ui64>::Id);
TestNumericType<float>(NUdf::TDataType<float>::Id);
TestNumericType<double>(NUdf::TDataType<double>::Id);
- }
-
- void TestOptionalNumericTypes() {
+ }
+
+ void TestOptionalNumericTypes() {
TestOptionalNumericType<bool>(NUdf::TDataType<bool>::Id);
TestOptionalNumericType<ui8>(NUdf::TDataType<ui8>::Id);
TestOptionalNumericType<i32>(NUdf::TDataType<i32>::Id);
@@ -62,77 +62,77 @@ protected:
TestOptionalNumericType<ui64>(NUdf::TDataType<ui64>::Id);
TestOptionalNumericType<float>(NUdf::TDataType<float>::Id);
TestOptionalNumericType<double>(NUdf::TDataType<double>::Id);
- }
-
- void TestStringTypes() {
+ }
+
+ void TestStringTypes() {
TestStringType(NUdf::TDataType<NUdf::TUtf8>::Id);
TestStringType(NUdf::TDataType<char*>::Id);
TestStringType(NUdf::TDataType<NUdf::TYson>::Id);
TestStringType(NUdf::TDataType<NUdf::TJson>::Id);
- }
-
- void TestOptionalStringTypes() {
+ }
+
+ void TestOptionalStringTypes() {
TestOptionalStringType(NUdf::TDataType<NUdf::TUtf8>::Id);
TestOptionalStringType(NUdf::TDataType<char*>::Id);
TestOptionalStringType(NUdf::TDataType<NUdf::TYson>::Id);
TestOptionalStringType(NUdf::TDataType<NUdf::TJson>::Id);
- }
-
- void TestListType() {
- TDefaultListRepresentation listValues;
- for (ui32 val: xrange(0, 10)) {
+ }
+
+ void TestListType() {
+ TDefaultListRepresentation listValues;
+ for (ui32 val: xrange(0, 10)) {
listValues = listValues.Append(NUdf::TUnboxedValuePod(val));
- }
+ }
TType* listType = PgmBuilder.NewListType(PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id));
const NUdf::TUnboxedValue value = HolderFactory.CreateDirectListHolder(std::move(listValues));
const auto uValue = TestPackUnpack(listType, value, "Type:List(ui32)");
-
- UNIT_ASSERT_VALUES_EQUAL(uValue.GetListLength(), 10);
- ui32 i = 0;
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetListLength(), 10);
+ ui32 i = 0;
const auto iter = uValue.GetListIterator();
for (NUdf::TUnboxedValue item; iter.Next(item); ++i) {
UNIT_ASSERT_VALUES_EQUAL(item.template Get<ui32>(), i);
- }
- UNIT_ASSERT_VALUES_EQUAL(i, 10);
- }
-
- void TestListOfOptionalsType() {
- TDefaultListRepresentation listValues;
- for (ui32 i: xrange(0, 10)) {
+ }
+ UNIT_ASSERT_VALUES_EQUAL(i, 10);
+ }
+
+ void TestListOfOptionalsType() {
+ TDefaultListRepresentation listValues;
+ for (ui32 i: xrange(0, 10)) {
NUdf::TUnboxedValue uVal = NUdf::TUnboxedValuePod();
- if (i % 2) {
+ if (i % 2) {
uVal = MakeString(TString(i * 2, '0' + i));
- }
+ }
listValues = listValues.Append(std::move(uVal));
- }
+ }
TType* listType = PgmBuilder.NewListType(PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)));
const NUdf::TUnboxedValue value = HolderFactory.CreateDirectListHolder(std::move(listValues));
const auto uValue = TestPackUnpack(listType, value, "Type:List(Optional(utf8))");
-
- UNIT_ASSERT_VALUES_EQUAL(uValue.GetListLength(), 10);
- ui32 i = 0;
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetListLength(), 10);
+ ui32 i = 0;
const auto iter = uValue.GetListIterator();
for (NUdf::TUnboxedValue uVal; iter.Next(uVal); ++i) {
- if (i % 2) {
+ if (i % 2) {
UNIT_ASSERT(uVal);
UNIT_ASSERT_VALUES_EQUAL(std::string_view(uVal.AsStringRef()), TString(i * 2, '0' + i));
- } else {
+ } else {
UNIT_ASSERT(!uVal);
- }
- }
- UNIT_ASSERT_VALUES_EQUAL(i, 10);
- }
-
- void TestTupleType() {
+ }
+ }
+ UNIT_ASSERT_VALUES_EQUAL(i, 10);
+ }
+
+ void TestTupleType() {
std::vector<TType*> tupleElemenTypes;
tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id));
- tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
- tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id));
- tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
- tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
- TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
-
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[3]));
+ TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
+
TUnboxedValueVector tupleElemens;
tupleElemens.push_back(MakeString("01234567890123456789"));
tupleElemens.push_back(MakeString("01234567890"));
@@ -140,39 +140,39 @@ protected:
tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
tupleElemens.push_back(NUdf::TUnboxedValuePod());
tupleElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
-
+
const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(tupleElemens);
const auto uValue = TestPackUnpack(tupleType, value, "Type:Tuple");
-
- {
+
+ {
auto e = uValue.GetElement(0);
UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789");
- }
- {
+ }
+ {
auto e = uValue.GetElement(1);
UNIT_ASSERT(e);
UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890");
- }
- {
+ }
+ {
auto e = uValue.GetElement(2);
UNIT_ASSERT(!e);
- }
- {
+ }
+ {
auto e = uValue.GetElement(3);
UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
- }
- {
+ }
+ {
auto e = uValue.GetElement(4);
UNIT_ASSERT(!e);
- }
- {
+ }
+ {
auto e = uValue.GetElement(5);
UNIT_ASSERT(e);
UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
- }
- }
-
- void TestStructType() {
+ }
+ }
+
+ void TestStructType() {
const std::vector<std::pair<std::string_view, TType*>> structElemenTypes = {
{"a", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)},
{"b", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)},
@@ -181,8 +181,8 @@ protected:
{"e", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id, true)},
{"f", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id, true)}
};
- TType* structType = PgmBuilder.NewStructType(structElemenTypes);
-
+ TType* structType = PgmBuilder.NewStructType(structElemenTypes);
+
TUnboxedValueVector structElemens;
structElemens.push_back(MakeString("01234567890123456789"));
structElemens.push_back(MakeString("01234567890"));
@@ -190,114 +190,114 @@ protected:
structElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
structElemens.push_back(NUdf::TUnboxedValuePod());
structElemens.push_back(NUdf::TUnboxedValuePod(ui64(12345)));
-
+
const NUdf::TUnboxedValue value = HolderFactory.VectorAsArray(structElemens);
const auto uValue = TestPackUnpack(structType, value, "Type:Struct");
-
- {
+
+ {
auto e = uValue.GetElement(0);
UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789");
- }
- {
+ }
+ {
auto e = uValue.GetElement(1);
UNIT_ASSERT(e);
UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890");
- }
- {
+ }
+ {
auto e = uValue.GetElement(2);
UNIT_ASSERT(!e);
- }
- {
+ }
+ {
auto e = uValue.GetElement(3);
UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
- }
- {
+ }
+ {
auto e = uValue.GetElement(4);
UNIT_ASSERT(!e);
- }
- {
+ }
+ {
auto e = uValue.GetElement(5);
UNIT_ASSERT(e);
UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345);
- }
- }
-
- void TestOptionalType() {
+ }
+ }
+
+ void TestOptionalType() {
TType* type = PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id);
- type = PgmBuilder.NewOptionalType(type);
- type = PgmBuilder.NewOptionalType(type);
- type = PgmBuilder.NewOptionalType(type);
-
+ type = PgmBuilder.NewOptionalType(type);
+ type = PgmBuilder.NewOptionalType(type);
+ type = PgmBuilder.NewOptionalType(type);
+
NUdf::TUnboxedValue uValue = NUdf::TUnboxedValuePod();
- uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:null");
+ uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:null");
UNIT_ASSERT(!uValue);
-
+
uValue = NUdf::TUnboxedValuePod(ui64(123));
- uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(opt(opt(123)))");
- UNIT_ASSERT_VALUES_EQUAL(uValue.Get<ui64>(), 123);
-
+ uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(opt(opt(123)))");
+ UNIT_ASSERT_VALUES_EQUAL(uValue.Get<ui64>(), 123);
+
uValue = NUdf::TUnboxedValuePod().MakeOptional().MakeOptional();
- uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(opt(null))");
+ uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(opt(null))");
UNIT_ASSERT(uValue);
- uValue = uValue.GetOptionalValue();
+ uValue = uValue.GetOptionalValue();
UNIT_ASSERT(uValue);
- uValue = uValue.GetOptionalValue();
+ uValue = uValue.GetOptionalValue();
UNIT_ASSERT(!uValue);
-
+
uValue = NUdf::TUnboxedValuePod().MakeOptional();
- uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(null)");
+ uValue = TestPackUnpack(type, uValue, "Type:Optional, Value:opt(null)");
UNIT_ASSERT(uValue);
- uValue = uValue.GetOptionalValue();
+ uValue = uValue.GetOptionalValue();
UNIT_ASSERT(!uValue);
- }
-
- void TestDictType() {
+ }
+
+ void TestDictType() {
TType* keyType = PgmBuilder.NewDataType(NUdf::TDataType<ui32>::Id);
TType* payloadType = PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id);
- TType* dictType = PgmBuilder.NewDictType(keyType, payloadType, false);
-
- TValuesDictHashSingleFixedMap<ui32> map;
+ TType* dictType = PgmBuilder.NewDictType(keyType, payloadType, false);
+
+ TValuesDictHashSingleFixedMap<ui32> map;
map[4] = NUdf::TUnboxedValuePod::Embedded("4");
map[10] = NUdf::TUnboxedValuePod::Embedded("10");
map[1] = NUdf::TUnboxedValuePod::Embedded("1");
const NUdf::TUnboxedValue value = HolderFactory.CreateDirectHashedSingleFixedMapHolder<ui32>(std::move(map));
-
+
const auto uValue = TestPackUnpack(dictType, value, "Type:Dict");
-
- UNIT_ASSERT_VALUES_EQUAL(uValue.GetDictLength(), 3);
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetDictLength(), 3);
const auto it = uValue.GetDictIterator();
for (NUdf::TUnboxedValue key, payload; it.NextPair(key, payload);) {
UNIT_ASSERT_VALUES_EQUAL(ToString(key.template Get<ui32>()), std::string_view(payload.AsStringRef()));
- }
- }
-
- void TestVariantTypeOverStruct() {
+ }
+ }
+
+ void TestVariantTypeOverStruct() {
const std::vector<std::pair<std::string_view, TType*>> structElemenTypes = {
{"a", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id)},
{"b", PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id, true)},
{"d", PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id)}
};
- TType* structType = PgmBuilder.NewStructType(structElemenTypes);
- TestVariantTypeImpl(PgmBuilder.NewVariantType(structType));
- }
-
- void TestVariantTypeOverTuple() {
+ TType* structType = PgmBuilder.NewStructType(structElemenTypes);
+ TestVariantTypeImpl(PgmBuilder.NewVariantType(structType));
+ }
+
+ void TestVariantTypeOverTuple() {
std::vector<TType*> tupleElemenTypes;
tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id));
- tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
+ tupleElemenTypes.push_back(PgmBuilder.NewOptionalType(tupleElemenTypes[0]));
tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<ui64>::Id));
- TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
- TestVariantTypeImpl(PgmBuilder.NewVariantType(tupleType));
- }
-
+ TType* tupleType = PgmBuilder.NewTupleType(tupleElemenTypes);
+ TestVariantTypeImpl(PgmBuilder.NewVariantType(tupleType));
+ }
+
void ValidateEmbeddedLength(const TStringBuf& buf, const TString& info) {
if (buf.size() > 8) {
UNIT_ASSERT_VALUES_EQUAL_C(*(const ui32*)buf.data() + 4, buf.size(), info);
- } else {
+ } else {
UNIT_ASSERT_VALUES_EQUAL_C(((ui32(*buf.data()) & 0x0f) >> 1) + 1, buf.size(), info);
- }
- }
-
+ }
+ }
+
void TestPackPerformance(TType* type, const NUdf::TUnboxedValuePod& uValue)
{
TValuePacker packer(false, type, UseCodegen);
@@ -309,55 +309,55 @@ protected:
NUdf::TUnboxedValue TestPackUnpack(TValuePacker& packer, const NUdf::TUnboxedValuePod& uValue,
const TString& additionalMsg, const std::optional<ui32>& expectedLength = {})
- {
- TStringBuf packedValue = packer.Pack(uValue);
- if (expectedLength) {
+ {
+ TStringBuf packedValue = packer.Pack(uValue);
+ if (expectedLength) {
UNIT_ASSERT_VALUES_EQUAL_C(packedValue.size(), *expectedLength, additionalMsg);
- }
- ValidateEmbeddedLength(packedValue, additionalMsg);
+ }
+ ValidateEmbeddedLength(packedValue, additionalMsg);
return packer.Unpack(packedValue, HolderFactory);
- }
-
+ }
+
NUdf::TUnboxedValue TestPackUnpack(TType* type, const NUdf::TUnboxedValuePod& uValue, const TString& additionalMsg,
const std::optional<ui32>& expectedLength = {})
- {
+ {
TValuePacker packer(false, type, UseCodegen);
- return TestPackUnpack(packer, uValue, additionalMsg, expectedLength);
- }
-
- template <typename T>
+ return TestPackUnpack(packer, uValue, additionalMsg, expectedLength);
+ }
+
+ template <typename T>
void TestNumericValue(T value, TValuePacker& packer, const TString& typeDesc) {
TString additionalMsg = TStringBuilder() << typeDesc << ", Value:" << value;
auto uValue = TestPackUnpack(packer, NUdf::TUnboxedValuePod(value), additionalMsg);
UNIT_ASSERT_VALUES_EQUAL_C(uValue.template Get<T>(), value, additionalMsg);
- }
-
- template <typename T>
+ }
+
+ template <typename T>
void TestNumericType(NUdf::TDataTypeId schemeType) {
TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
TValuePacker packer(false, PgmBuilder.NewDataType(schemeType), UseCodegen);
-
- TestNumericValue<T>(Max<T>(), packer, typeDesc);
- TestNumericValue<T>(Min<T>(), packer, typeDesc);
- TestNumericValue<T>(T(0), packer, typeDesc);
- TestNumericValue<T>(T(1), packer, typeDesc);
- }
-
- template <typename T>
+
+ TestNumericValue<T>(Max<T>(), packer, typeDesc);
+ TestNumericValue<T>(Min<T>(), packer, typeDesc);
+ TestNumericValue<T>(T(0), packer, typeDesc);
+ TestNumericValue<T>(T(1), packer, typeDesc);
+ }
+
+ template <typename T>
void TestOptionalNumericValue(std::optional<T> value, TValuePacker& packer, const TString& typeDesc,
const std::optional<ui32>& expectedLength = {})
- {
+ {
TString additionalMsg = TStringBuilder() << typeDesc << "), Value:" << (value ? ToString(*value) : TString("null"));
const auto v = value ? NUdf::TUnboxedValuePod(*value) : NUdf::TUnboxedValuePod();
const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength);
- if (value) {
+ if (value) {
UNIT_ASSERT_VALUES_EQUAL_C(uValue.template Get<T>(), *value, additionalMsg);
- } else {
+ } else {
UNIT_ASSERT_C(!uValue, additionalMsg);
- }
- }
-
- template <typename T>
+ }
+ }
+
+ template <typename T>
void TestOptionalNumericType(NUdf::TDataTypeId schemeType) {
TString typeDesc = TStringBuilder() << ", Type:Optional(" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
TValuePacker packer(false, PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(schemeType)), UseCodegen);
@@ -366,15 +366,15 @@ protected:
TestOptionalNumericValue<T>(std::optional<T>(), packer, typeDesc, 1);
TestOptionalNumericValue<T>(std::optional<T>(0), packer, typeDesc);
TestOptionalNumericValue<T>(std::optional<T>(1), packer, typeDesc);
- }
-
+ }
+
void TestStringValue(const std::string_view& value, TValuePacker& packer, const TString& typeDesc, ui32 expectedLength) {
TString additionalMsg = TStringBuilder() << typeDesc << ", Value:" << value;
const auto v = NUdf::TUnboxedValue(MakeString(value));
const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength);
UNIT_ASSERT_VALUES_EQUAL_C(std::string_view(uValue.AsStringRef()), value, additionalMsg);
- }
-
+ }
+
void TestStringType(NUdf::TDataTypeId schemeType) {
TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
TValuePacker packer(false, PgmBuilder.NewDataType(schemeType), UseCodegen);
@@ -383,8 +383,8 @@ protected:
TestStringValue("1234567", packer, typeDesc, 7 + 1);
TestStringValue("", packer, typeDesc, 1);
TestStringValue("12345678", packer, typeDesc, 8 + 4);
- }
-
+ }
+
void TestUuidType() {
auto schemeType = NUdf::TDataType<NUdf::TUuid>::Id;
TString typeDesc = TStringBuilder() << ", Type:" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
@@ -396,13 +396,13 @@ protected:
TString additionalMsg = TStringBuilder() << typeDesc << "), Value:" << (value ? *value : TString("null"));
const auto v = value ? NUdf::TUnboxedValue(MakeString(*value)) : NUdf::TUnboxedValue();
const auto uValue = TestPackUnpack(packer, v, additionalMsg, expectedLength);
- if (value) {
+ if (value) {
UNIT_ASSERT_VALUES_EQUAL_C(std::string_view(uValue.AsStringRef()), *value, additionalMsg);
- } else {
+ } else {
UNIT_ASSERT_C(!uValue, additionalMsg);
- }
- }
-
+ }
+ }
+
void TestOptionalStringType(NUdf::TDataTypeId schemeType) {
TString typeDesc = TStringBuilder() << ", Type:Optional(" << NUdf::GetDataTypeInfo(NUdf::GetDataSlot(schemeType)).Name;
TValuePacker packer(false, PgmBuilder.NewOptionalType(PgmBuilder.NewDataType(schemeType)), UseCodegen);
@@ -412,37 +412,37 @@ protected:
TestOptionalStringValue("1234567", packer, typeDesc, 7 + 1);
TestOptionalStringValue("", packer, typeDesc, 1);
TestOptionalStringValue("12345678", packer, typeDesc, 8 + 4);
- }
-
- void TestVariantTypeImpl(TType* variantType) {
+ }
+
+ void TestVariantTypeImpl(TType* variantType) {
TString descr = TStringBuilder() << "Type:Variant("
- << static_cast<TVariantType*>(variantType)->GetUnderlyingType()->GetKindAsStr() << ')';
- {
+ << static_cast<TVariantType*>(variantType)->GetUnderlyingType()->GetKindAsStr() << ')';
+ {
const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(MakeString("01234567890123456789"), 0);
const auto uValue = TestPackUnpack(variantType, value, descr);
-
- UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 0);
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 0);
auto e = uValue.GetVariantItem();
UNIT_ASSERT_VALUES_EQUAL(std::string_view(e.AsStringRef()), "01234567890123456789");
- }
- {
+ }
+ {
const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(), 1);
const auto uValue = TestPackUnpack(variantType, value, descr);
-
- UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 1);
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 1);
auto e = uValue.GetVariantItem();
UNIT_ASSERT(!e);
- }
- {
+ }
+ {
const NUdf::TUnboxedValue value = HolderFactory.CreateVariantHolder(NUdf::TUnboxedValuePod(ui64(12345)), 2);
const auto uValue = TestPackUnpack(variantType, value, descr);
-
- UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 2);
+
+ UNIT_ASSERT_VALUES_EQUAL(uValue.GetVariantIndex(), 2);
auto e = uValue.GetVariantItem();
UNIT_ASSERT_VALUES_EQUAL(e.template Get<ui64>(), 12345ull);
- }
- }
-
+ }
+ }
+
void TestTuplePackPerformance() {
std::vector<TType*> tupleElemenTypes;
tupleElemenTypes.push_back(PgmBuilder.NewDataType(NUdf::TDataType<NUdf::TUtf8>::Id));
@@ -491,16 +491,16 @@ protected:
TestPackPerformance(type, v);
}
-private:
- TIntrusivePtr<NMiniKQL::IFunctionRegistry> FunctionRegistry;
- TIntrusivePtr<IRandomProvider> RandomProvider;
+private:
+ TIntrusivePtr<NMiniKQL::IFunctionRegistry> FunctionRegistry;
+ TIntrusivePtr<IRandomProvider> RandomProvider;
TScopedAlloc Alloc;
- TTypeEnvironment Env;
- TProgramBuilder PgmBuilder;
+ TTypeEnvironment Env;
+ TProgramBuilder PgmBuilder;
TMemoryUsageInfo MemInfo;
- THolderFactory HolderFactory;
-};
-
+ THolderFactory HolderFactory;
+};
+
class TMiniKQLComputationNodePackCodegenTest: public TMiniKQLComputationNodePackTest<true> {
UNIT_TEST_SUITE(TMiniKQLComputationNodePackCodegenTest);
UNIT_TEST(TestNumericTypes);
@@ -521,7 +521,7 @@ class TMiniKQLComputationNodePackCodegenTest: public TMiniKQLComputationNodePack
UNIT_TEST(TestTuplePackPerformance);
UNIT_TEST_SUITE_END();
};
-
+
class TMiniKQLComputationNodePackBySwitchTest: public TMiniKQLComputationNodePackTest<false> {
UNIT_TEST_SUITE(TMiniKQLComputationNodePackBySwitchTest);
UNIT_TEST(TestNumericTypes);
@@ -549,5 +549,5 @@ UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodePackCodegenTest);
#endif
UNIT_TEST_SUITE_REGISTRATION(TMiniKQLComputationNodePackBySwitchTest);
-}
-}
+}
+}
diff --git a/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp b/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp
index ed2f12303a..bf0f1d0b0d 100644
--- a/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp
+++ b/ydb/library/yql/minikql/computation/mkql_validate_ut.cpp
@@ -649,7 +649,7 @@ Y_UNIT_TEST_SUITE(TMiniKQLValidateTest) {
FullValidateValueFunc fullValidateFunc = FullValidateValueFunc(),
NUdf::EValidateMode validateMode = NUdf::EValidateMode::Lazy) {
TScopedAlloc alloc;
- TTypeEnvironment env(alloc);
+ TTypeEnvironment env(alloc);
NUdf::ITypeInfoHelper::TPtr typeInfoHelper(new TTypeInfoHelper);
auto functionRegistry = CreateFunctionRegistryWithUDFs();
auto randomProvider = CreateDeterministicRandomProvider(1);
diff --git a/ydb/library/yql/minikql/computation/ut/ya.make b/ydb/library/yql/minikql/computation/ut/ya.make
index ab27689c7b..2aadc4dfc7 100644
--- a/ydb/library/yql/minikql/computation/ut/ya.make
+++ b/ydb/library/yql/minikql/computation/ut/ya.make
@@ -17,9 +17,9 @@ OWNER(
)
SRCS(
- mkql_computation_node_pack_ut.cpp
+ mkql_computation_node_pack_ut.cpp
mkql_computation_node_list_ut.cpp
- mkql_computation_node_dict_ut.cpp
+ mkql_computation_node_dict_ut.cpp
mkql_computation_node_graph_saveload_ut.cpp
mkql_validate_ut.cpp
mkql_value_builder_ut.cpp
diff --git a/ydb/library/yql/minikql/dom/convert.h b/ydb/library/yql/minikql/dom/convert.h
index c2d79c72d6..27bb99d61f 100644
--- a/ydb/library/yql/minikql/dom/convert.h
+++ b/ydb/library/yql/minikql/dom/convert.h
@@ -173,7 +173,7 @@ template<bool Strict, bool AutoConvert, typename TargetType>
TUnboxedValuePod ConvertToFloat(TUnboxedValuePod x, const IValueBuilder* valueBuilder, const TSourcePosition& pos) {
switch (GetNodeType(x)) {
case ENodeType::Double:
- return TUnboxedValuePod(TargetType(x.Get<double>()));
+ return TUnboxedValuePod(TargetType(x.Get<double>()));
case ENodeType::Uint64:
return TUnboxedValuePod(TargetType(x.Get<ui64>()));
case ENodeType::Int64:
@@ -260,7 +260,7 @@ TUnboxedValuePod ConvertToString(TUnboxedValuePod x, const IValueBuilder* valueB
return {};
case ENodeType::Double:
if constexpr (AutoConvert)
- return valueBuilder->NewString(::FloatToString(x.Get<double>())).Release();
+ return valueBuilder->NewString(::FloatToString(x.Get<double>())).Release();
else if constexpr (Strict)
break;
else
diff --git a/ydb/library/yql/minikql/dom/peel.cpp b/ydb/library/yql/minikql/dom/peel.cpp
index 33f22011dd..81bd99c56c 100644
--- a/ydb/library/yql/minikql/dom/peel.cpp
+++ b/ydb/library/yql/minikql/dom/peel.cpp
@@ -313,7 +313,7 @@ TUnboxedValuePod PeelOptional(const ITypeInfoHelper::TPtr typeHelper, const TTyp
if (const auto result = TryPeelDom<Strict, AutoConvert>(typeHelper, itemType, value, valueBuilder, pos); AutoConvert || result)
return result;
else if constexpr (Strict)
- UdfTerminate("Failed to convert Yson DOM.");
+ UdfTerminate("Failed to convert Yson DOM.");
else
return TUnboxedValuePod().MakeOptional();
}
diff --git a/ydb/library/yql/minikql/mkql_alloc.cpp b/ydb/library/yql/minikql/mkql_alloc.cpp
index 9b77147a26..8a3953dd4c 100644
--- a/ydb/library/yql/minikql/mkql_alloc.cpp
+++ b/ydb/library/yql/minikql/mkql_alloc.cpp
@@ -1,12 +1,12 @@
-#include "mkql_alloc.h"
+#include "mkql_alloc.h"
#include <util/system/align.h>
#include <ydb/library/yql/public/udf/udf_value.h>
#include <tuple>
-
-namespace NKikimr {
-
-namespace NMiniKQL {
-
+
+namespace NKikimr {
+
+namespace NMiniKQL {
+
Y_POD_THREAD(TAllocState*) TlsAllocState;
TAllocPageHeader TAllocState::EmptyPageHeader = { 0, 0, 0, 0, 0, nullptr };
@@ -127,6 +127,6 @@ void TPagedArena::Clear() noexcept {
CurrentPage_ = &TAllocState::EmptyPageHeader;
}
-} // NMiniKQL
-
-} // NKikimr
+} // NMiniKQL
+
+} // NKikimr
diff --git a/ydb/library/yql/minikql/mkql_alloc.h b/ydb/library/yql/minikql/mkql_alloc.h
index 5e864cdc4c..b4cc546136 100644
--- a/ydb/library/yql/minikql/mkql_alloc.h
+++ b/ydb/library/yql/minikql/mkql_alloc.h
@@ -1,18 +1,18 @@
-#pragma once
+#pragma once
#include "aligned_page_pool.h"
#include "mkql_mem_info.h"
#include <ydb/library/yql/public/udf/udf_allocator.h>
#include <ydb/library/yql/public/udf/udf_value.h>
-#include <util/system/defaults.h>
+#include <util/system/defaults.h>
#include <util/system/tls.h>
#include <util/system/align.h>
-#include <new>
+#include <new>
#include <unordered_map>
-
-namespace NKikimr {
-
-namespace NMiniKQL {
-
+
+namespace NKikimr {
+
+namespace NMiniKQL {
+
const ui64 MKQL_ALIGNMENT = 16;
struct TAllocPageHeader {
@@ -236,7 +236,7 @@ private:
void* AllocSlow(size_t sz);
private:
- TAlignedPagePool* PagePool_;
+ TAlignedPagePool* PagePool_;
TAllocPageHeader* CurrentPage_ = &TAllocState::EmptyPageHeader;
};
@@ -332,22 +332,22 @@ inline void MKQLUnregisterObject(NUdf::TBoxedValue* value) noexcept {
return value->Unlink();
}
-struct TWithMiniKQLAlloc {
- void* operator new(size_t sz) {
+struct TWithMiniKQLAlloc {
+ void* operator new(size_t sz) {
return MKQLAllocWithSize(sz);
- }
-
- void* operator new[](size_t sz) {
+ }
+
+ void* operator new[](size_t sz) {
return MKQLAllocWithSize(sz);
- }
-
+ }
+
void operator delete(void *mem, std::size_t sz) noexcept {
MKQLFreeWithSize(mem, sz);
- }
-
+ }
+
void operator delete[](void *mem, std::size_t sz) noexcept {
MKQLFreeWithSize(mem, sz);
- }
+ }
};
template <typename T, typename... Args>
@@ -610,6 +610,6 @@ private:
};
-} // NMiniKQL
-
-} // NKikimr
+} // NMiniKQL
+
+} // NKikimr
diff --git a/ydb/library/yql/minikql/mkql_alloc_ut.cpp b/ydb/library/yql/minikql/mkql_alloc_ut.cpp
index 72d8f696af..6c86dbdcbb 100644
--- a/ydb/library/yql/minikql/mkql_alloc_ut.cpp
+++ b/ydb/library/yql/minikql/mkql_alloc_ut.cpp
@@ -7,7 +7,7 @@ namespace NMiniKQL {
Y_UNIT_TEST_SUITE(TMiniKQLAllocTest) {
Y_UNIT_TEST(TestPagedArena) {
- TAlignedPagePool pagePool;
+ TAlignedPagePool pagePool;
{
TPagedArena arena(&pagePool);
diff --git a/ydb/library/yql/minikql/mkql_function_registry.cpp b/ydb/library/yql/minikql/mkql_function_registry.cpp
index d3545ad138..a1ef87f9ca 100644
--- a/ydb/library/yql/minikql/mkql_function_registry.cpp
+++ b/ydb/library/yql/minikql/mkql_function_registry.cpp
@@ -262,10 +262,10 @@ public:
<< ", function not found";
}
- if (funcInfo->ModuleIRUniqID) {
- funcInfo->ModuleIRUniqID.prepend(moduleName);
- }
-
+ if (funcInfo->ModuleIRUniqID) {
+ funcInfo->ModuleIRUniqID.prepend(moduleName);
+ }
+
return TStatus::Ok();
}
@@ -488,7 +488,7 @@ void FindUdfsInDir(const TString& dirPath, TVector<TString>* paths)
}
paths->push_back(std::move(path));
- }
+ }
}
}
}
diff --git a/ydb/library/yql/minikql/mkql_mem_info.h b/ydb/library/yql/minikql/mkql_mem_info.h
index 8e880d1db3..a9586eda80 100644
--- a/ydb/library/yql/minikql/mkql_mem_info.h
+++ b/ydb/library/yql/minikql/mkql_mem_info.h
@@ -67,9 +67,9 @@ public:
inline void Take(const void* mem, ui64 size, TString location) {
Allocated_ += size;
Peak_ = Max(Peak_, Allocated_ - Freed_);
- if (size == 0) {
- return;
- }
+ if (size == 0) {
+ return;
+ }
if (AllowMissing_) {
auto it = AllocationsMap_.find(mem);
if (it != AllocationsMap_.end() && it->second.IsDeleted) {
@@ -86,9 +86,9 @@ public:
#ifndef NDEBUG
inline void Return(const void* mem, ui64 size) {
Freed_ += size;
- if (size == 0) {
- return;
- }
+ if (size == 0) {
+ return;
+ }
//Clog << Title_ << " free: " << size << " -> " << mem << " " << AllocationsMap_.size() << Endl;
auto it = AllocationsMap_.find(mem);
if (AllowMissing_ && it == AllocationsMap_.end()) {
diff --git a/ydb/library/yql/minikql/mkql_node.cpp b/ydb/library/yql/minikql/mkql_node.cpp
index 4c6f3e45ad..be7f1bdb8d 100644
--- a/ydb/library/yql/minikql/mkql_node.cpp
+++ b/ydb/library/yql/minikql/mkql_node.cpp
@@ -15,7 +15,7 @@ namespace NMiniKQL {
using namespace NDetail;
-TTypeEnvironment::TTypeEnvironment(TScopedAlloc& alloc)
+TTypeEnvironment::TTypeEnvironment(TScopedAlloc& alloc)
: Alloc(alloc)
, Arena(&Alloc.Ref())
, EmptyStruct(nullptr)
diff --git a/ydb/library/yql/minikql/mkql_node.h b/ydb/library/yql/minikql/mkql_node.h
index ee74455ea6..ece683a3a0 100644
--- a/ydb/library/yql/minikql/mkql_node.h
+++ b/ydb/library/yql/minikql/mkql_node.h
@@ -1,7 +1,7 @@
#pragma once
#include "defs.h"
#include "mkql_mem_info.h"
-#include "mkql_alloc.h"
+#include "mkql_alloc.h"
#include <ydb/library/yql/public/udf/udf_value.h>
@@ -354,7 +354,7 @@ namespace NMiniKQL {
class TTypeEnvironment : private TNonCopyable {
public:
- explicit TTypeEnvironment(TScopedAlloc& alloc);
+ explicit TTypeEnvironment(TScopedAlloc& alloc);
~TTypeEnvironment();
diff --git a/ydb/library/yql/minikql/mkql_node_serialization.cpp b/ydb/library/yql/minikql/mkql_node_serialization.cpp
index e314f38cd6..8eb835bead 100644
--- a/ydb/library/yql/minikql/mkql_node_serialization.cpp
+++ b/ydb/library/yql/minikql/mkql_node_serialization.cpp
@@ -4,10 +4,10 @@
#include "mkql_type_ops.h"
#include <ydb/library/yql/minikql/pack_num.h>
-
+
#include <library/cpp/containers/stack_vector/stack_vec.h>
#include <library/cpp/packedtypes/zigzag.h>
-
+
#include <util/generic/algorithm.h>
#include <util/system/unaligned_mem.h>
@@ -947,13 +947,13 @@ namespace {
}
Y_FORCE_INLINE void WriteVar32(ui32 value) {
- char buf[MAX_PACKED32_SIZE];
- Out.AppendNoAlias(buf, Pack32(value, buf));
+ char buf[MAX_PACKED32_SIZE];
+ Out.AppendNoAlias(buf, Pack32(value, buf));
}
Y_FORCE_INLINE void WriteVar64(ui64 value) {
- char buf[MAX_PACKED64_SIZE];
- Out.AppendNoAlias(buf, Pack64(value, buf));
+ char buf[MAX_PACKED64_SIZE];
+ Out.AppendNoAlias(buf, Pack64(value, buf));
}
private:
@@ -1070,21 +1070,21 @@ namespace {
Y_FORCE_INLINE ui32 ReadVar32() {
ui32 result = 0;
- size_t count = Unpack32(Current, End - Current, result);
- if (!count) {
+ size_t count = Unpack32(Current, End - Current, result);
+ if (!count) {
ThrowCorrupted();
- }
- Current += count;
+ }
+ Current += count;
return result;
}
Y_FORCE_INLINE ui64 ReadVar64() {
ui64 result = 0;
- size_t count = Unpack64(Current, End - Current, result);
- if (!count) {
+ size_t count = Unpack64(Current, End - Current, result);
+ if (!count) {
ThrowCorrupted();
- }
- Current += count;
+ }
+ Current += count;
return result;
}
diff --git a/ydb/library/yql/minikql/mkql_program_builder.cpp b/ydb/library/yql/minikql/mkql_program_builder.cpp
index 47ae6e4c8f..89e8546c31 100644
--- a/ydb/library/yql/minikql/mkql_program_builder.cpp
+++ b/ydb/library/yql/minikql/mkql_program_builder.cpp
@@ -913,7 +913,7 @@ TRuntimeNode TProgramBuilder::OrderedFlatMap(TRuntimeNode list, const TUnaryLamb
}
TRuntimeNode TProgramBuilder::Filter(TRuntimeNode list, const TUnaryLambda& handler)
-{
+{
return BuildFilter(__func__, list, handler);
}
@@ -2110,7 +2110,7 @@ TRuntimeNode TProgramBuilder::NewVariant(TRuntimeNode item, const std::string_vi
}
TRuntimeNode TProgramBuilder::Coalesce(TRuntimeNode data, TRuntimeNode defaultData) {
- bool isOptional = false;
+ bool isOptional = false;
const auto dataType = UnpackOptional(data, isOptional);
if (!isOptional) {
MKQL_ENSURE(data.GetStaticType()->IsSameType(*defaultData.GetStaticType()), "Mismatch operand types");
@@ -2495,7 +2495,7 @@ TRuntimeNode TProgramBuilder::If(TRuntimeNode condition, TRuntimeNode thenBranch
const bool isOptional = condOpt || thenOpt || elseOpt;
TCallableBuilder callableBuilder(Env, __func__, isOptional ? NewOptionalType(thenUnpacked) : thenUnpacked);
- callableBuilder.Add(condition);
+ callableBuilder.Add(condition);
callableBuilder.Add(thenBranch);
callableBuilder.Add(elseBranch);
return TRuntimeNode(callableBuilder.Build(), false);
@@ -2977,18 +2977,18 @@ TRuntimeNode TProgramBuilder::Way(TRuntimeNode variant) {
return TRuntimeNode(callableBuilder.Build(), false);
}
-TRuntimeNode TProgramBuilder::VariantItem(TRuntimeNode variant) {
- bool isOptional;
- auto unpacked = UnpackOptional(variant, isOptional);
- auto type = AS_TYPE(TVariantType, unpacked);
- auto underlyingType = type->GetAlternativeType(0);
- auto resType = isOptional ? TOptionalType::Create(underlyingType, Env) : underlyingType;
-
+TRuntimeNode TProgramBuilder::VariantItem(TRuntimeNode variant) {
+ bool isOptional;
+ auto unpacked = UnpackOptional(variant, isOptional);
+ auto type = AS_TYPE(TVariantType, unpacked);
+ auto underlyingType = type->GetAlternativeType(0);
+ auto resType = isOptional ? TOptionalType::Create(underlyingType, Env) : underlyingType;
+
TCallableBuilder callableBuilder(Env, __func__, resType);
- callableBuilder.Add(variant);
- return TRuntimeNode(callableBuilder.Build(), false);
-}
-
+ callableBuilder.Add(variant);
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
TRuntimeNode TProgramBuilder::VisitAll(TRuntimeNode variant, std::function<TRuntimeNode(ui32, TRuntimeNode)> handler) {
const auto type = AS_TYPE(TVariantType, variant);
std::vector<TRuntimeNode> items;
@@ -4413,38 +4413,38 @@ TRuntimeNode TProgramBuilder::WideCondense1(TRuntimeNode flow, const TWideLambda
return TRuntimeNode(callableBuilder.Build(), false);
}
-TRuntimeNode TProgramBuilder::CombineCore(TRuntimeNode stream,
+TRuntimeNode TProgramBuilder::CombineCore(TRuntimeNode stream,
const TUnaryLambda& keyExtractor,
const TBinaryLambda& init,
const TTernaryLambda& update,
const TBinaryLambda& finish,
- ui64 memLimit)
-{
+ ui64 memLimit)
+{
if constexpr (RuntimeVersion < 3U) {
THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__;
}
-
+
const bool isStream = stream.GetStaticType()->IsStream();
const auto itemType = isStream ? AS_TYPE(TStreamType, stream)->GetItemType() : AS_TYPE(TFlowType, stream)->GetItemType();
-
+
const auto itemArg = Arg(itemType);
-
+
const auto key = keyExtractor(itemArg);
const auto keyType = key.GetStaticType();
-
+
const auto keyArg = Arg(keyType);
-
+
const auto stateInit = init(keyArg, itemArg);
const auto stateType = stateInit.GetStaticType();
-
+
const auto stateArg = Arg(stateType);
-
+
const auto stateUpdate = update(keyArg, itemArg, stateArg);
const auto finishItem = finish(keyArg, stateArg);
-
+
const auto finishType = finishItem.GetStaticType();
MKQL_ENSURE(finishType->IsList() || finishType->IsStream() || finishType->IsOptional(), "Expected list, stream or optional");
-
+
TType* retItemType = nullptr;
if (finishType->IsOptional()) {
retItemType = AS_TYPE(TOptionalType, finishType)->GetItemType();
@@ -4453,7 +4453,7 @@ TRuntimeNode TProgramBuilder::CombineCore(TRuntimeNode stream,
} else if (finishType->IsStream()) {
retItemType = AS_TYPE(TStreamType, finishType)->GetItemType();
}
-
+
const auto resultStreamType = isStream ? NewStreamType(retItemType) : NewFlowType(retItemType);
TCallableBuilder callableBuilder(Env, __func__, resultStreamType);
callableBuilder.Add(stream);
@@ -4467,56 +4467,56 @@ TRuntimeNode TProgramBuilder::CombineCore(TRuntimeNode stream,
callableBuilder.Add(NewDataLiteral(memLimit));
return TRuntimeNode(callableBuilder.Build(), false);
-}
-
-TRuntimeNode TProgramBuilder::GroupingCore(TRuntimeNode stream,
+}
+
+TRuntimeNode TProgramBuilder::GroupingCore(TRuntimeNode stream,
const TBinaryLambda& groupSwitch,
- const TUnaryLambda& keyExtractor,
- const TUnaryLambda& handler)
-{
- if (handler && RuntimeVersion < 20U) {
- THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__ << " with handler";
- }
-
- auto itemType = AS_TYPE(TStreamType, stream)->GetItemType();
-
+ const TUnaryLambda& keyExtractor,
+ const TUnaryLambda& handler)
+{
+ if (handler && RuntimeVersion < 20U) {
+ THROW yexception() << "Runtime version (" << RuntimeVersion << ") too old for " << __func__ << " with handler";
+ }
+
+ auto itemType = AS_TYPE(TStreamType, stream)->GetItemType();
+
TRuntimeNode keyExtractorItemArg = Arg(itemType);
TRuntimeNode keyExtractorResult = keyExtractor(keyExtractorItemArg);
-
+
TRuntimeNode groupSwitchKeyArg = Arg(keyExtractorResult.GetStaticType());
TRuntimeNode groupSwitchItemArg = Arg(itemType);
-
+
TRuntimeNode groupSwitchResult = groupSwitch(groupSwitchKeyArg, groupSwitchItemArg);
MKQL_ENSURE(AS_TYPE(TDataType, groupSwitchResult)->GetSchemeType() == NUdf::TDataType<bool>::Id,
"Expected bool type");
- TRuntimeNode handlerItemArg;
- TRuntimeNode handlerResult;
-
- if (handler) {
- handlerItemArg = Arg(itemType);
- handlerResult = handler(handlerItemArg);
- itemType = handlerResult.GetStaticType();
- }
-
- const std::array<TType*, 2U> tupleItems = {{ keyExtractorResult.GetStaticType(), NewStreamType(itemType) }};
+ TRuntimeNode handlerItemArg;
+ TRuntimeNode handlerResult;
+
+ if (handler) {
+ handlerItemArg = Arg(itemType);
+ handlerResult = handler(handlerItemArg);
+ itemType = handlerResult.GetStaticType();
+ }
+
+ const std::array<TType*, 2U> tupleItems = {{ keyExtractorResult.GetStaticType(), NewStreamType(itemType) }};
const auto finishType = NewStreamType(NewTupleType(tupleItems));
-
+
TCallableBuilder callableBuilder(Env, __func__, finishType);
- callableBuilder.Add(stream);
+ callableBuilder.Add(stream);
callableBuilder.Add(keyExtractorResult);
callableBuilder.Add(groupSwitchResult);
callableBuilder.Add(keyExtractorItemArg);
callableBuilder.Add(groupSwitchKeyArg);
callableBuilder.Add(groupSwitchItemArg);
- if (handler) {
- callableBuilder.Add(handlerResult);
- callableBuilder.Add(handlerItemArg);
- }
-
- return TRuntimeNode(callableBuilder.Build(), false);
-}
-
+ if (handler) {
+ callableBuilder.Add(handlerResult);
+ callableBuilder.Add(handlerItemArg);
+ }
+
+ return TRuntimeNode(callableBuilder.Build(), false);
+}
+
TRuntimeNode TProgramBuilder::Chopper(TRuntimeNode flow, const TUnaryLambda& keyExtractor, const TBinaryLambda& groupSwitch, const TBinaryLambda& groupHandler) {
const auto flowType = flow.GetStaticType();
MKQL_ENSURE(flowType->IsFlow() || flowType->IsStream(), "Expected flow or stream.");
diff --git a/ydb/library/yql/minikql/mkql_program_builder.h b/ydb/library/yql/minikql/mkql_program_builder.h
index 10f1ad7ccf..89ff55b999 100644
--- a/ydb/library/yql/minikql/mkql_program_builder.h
+++ b/ydb/library/yql/minikql/mkql_program_builder.h
@@ -406,16 +406,16 @@ public:
ui64 memLimit, std::optional<ui32> sortedTableOrder,
EAnyJoinSettings anyJoinSettings, const ui32 tableIndexField,
TType* returnType);
- TRuntimeNode CombineCore(TRuntimeNode stream,
+ TRuntimeNode CombineCore(TRuntimeNode stream,
const TUnaryLambda& keyExtractor,
const TBinaryLambda& init,
const TTernaryLambda& update,
const TBinaryLambda& finish,
- ui64 memLimit);
- TRuntimeNode GroupingCore(TRuntimeNode stream,
+ ui64 memLimit);
+ TRuntimeNode GroupingCore(TRuntimeNode stream,
const TBinaryLambda& groupSwitch,
- const TUnaryLambda& keyExtractor,
- const TUnaryLambda& handler = {});
+ const TUnaryLambda& keyExtractor,
+ const TUnaryLambda& handler = {});
TRuntimeNode HoppingCore(TRuntimeNode list,
const TUnaryLambda& timeExtractor,
const TUnaryLambda& init,
@@ -564,7 +564,7 @@ public:
TRuntimeNode Guess(TRuntimeNode variant, const std::string_view& memberName);
TRuntimeNode VisitAll(TRuntimeNode variant, std::function<TRuntimeNode(ui32, TRuntimeNode)> handler);
TRuntimeNode Way(TRuntimeNode variant);
- TRuntimeNode VariantItem(TRuntimeNode variant);
+ TRuntimeNode VariantItem(TRuntimeNode variant);
//-- random functions
// expects ui64 seed, returns resource
diff --git a/ydb/library/yql/minikql/mkql_type_builder.cpp b/ydb/library/yql/minikql/mkql_type_builder.cpp
index 4aecb564bf..76bbeb1827 100644
--- a/ydb/library/yql/minikql/mkql_type_builder.cpp
+++ b/ydb/library/yql/minikql/mkql_type_builder.cpp
@@ -1267,16 +1267,16 @@ NUdf::IFunctionTypeInfoBuilder1& TFunctionTypeInfoBuilder::ImplementationImpl(
}
NUdf::IFunctionTypeInfoBuilder7& TFunctionTypeInfoBuilder::IRImplementationImpl(
- const NUdf::TStringRef& moduleIR,
- const NUdf::TStringRef& moduleIRUniqId,
- const NUdf::TStringRef& functionName
-) {
- ModuleIR_ = moduleIR;
- ModuleIRUniqID_ = moduleIRUniqId;
- IRFunctionName_ = functionName;
- return *this;
-}
-
+ const NUdf::TStringRef& moduleIR,
+ const NUdf::TStringRef& moduleIRUniqId,
+ const NUdf::TStringRef& functionName
+) {
+ ModuleIR_ = moduleIR;
+ ModuleIRUniqID_ = moduleIRUniqId;
+ IRFunctionName_ = functionName;
+ return *this;
+}
+
NUdf::TType* TFunctionTypeInfoBuilder::Null() const {
return Env_.GetTypeOfNull();
}
@@ -1427,9 +1427,9 @@ void TFunctionTypeInfoBuilder::Build(TFunctionTypeInfo* funcInfo)
funcInfo->RunConfigType = RunConfigType_;
funcInfo->UserType = UserType_;
funcInfo->Implementation = std::move(Implementation_);
- funcInfo->ModuleIR = std::move(ModuleIR_);
- funcInfo->ModuleIRUniqID = std::move(ModuleIRUniqID_);
- funcInfo->IRFunctionName = std::move(IRFunctionName_);
+ funcInfo->ModuleIR = std::move(ModuleIR_);
+ funcInfo->ModuleIRUniqID = std::move(ModuleIRUniqID_);
+ funcInfo->IRFunctionName = std::move(IRFunctionName_);
funcInfo->BlockImplementation = std::move(BlockImplementation_);
}
diff --git a/ydb/library/yql/minikql/mkql_type_builder.h b/ydb/library/yql/minikql/mkql_type_builder.h
index a93fb41cee..6d070284d3 100644
--- a/ydb/library/yql/minikql/mkql_type_builder.h
+++ b/ydb/library/yql/minikql/mkql_type_builder.h
@@ -17,9 +17,9 @@ struct TFunctionTypeInfo
const TType* UserType = nullptr;
NUdf::TUniquePtr<NUdf::IBoxedValue> Implementation;
NUdf::TUniquePtr<NUdf::IBoxedValue> BlockImplementation;
- TString ModuleIR;
- TString ModuleIRUniqID;
- TString IRFunctionName;
+ TString ModuleIR;
+ TString ModuleIRUniqID;
+ TString IRFunctionName;
bool Deterministic = true;
};
@@ -100,13 +100,13 @@ public:
NUdf::ICompare::TPtr MakeCompare(const NUdf::TType* type) override;
NUdf::TType* Decimal(ui8 precision, ui8 scale) const override;
-
+
NUdf::IFunctionTypeInfoBuilder7& IRImplementationImpl(
- const NUdf::TStringRef& moduleIR,
- const NUdf::TStringRef& moduleIRUniqId,
- const NUdf::TStringRef& functionName
- ) override;
-
+ const NUdf::TStringRef& moduleIR,
+ const NUdf::TStringRef& moduleIRUniqId,
+ const NUdf::TStringRef& functionName
+ ) override;
+
NUdf::TType* Null() const override;
NUdf::TType* EmptyList() const override;
NUdf::TType* EmptyDict() const override;
@@ -135,9 +135,9 @@ private:
NUdf::ICountersProvider* CountersProvider_;
NUdf::TSourcePosition Pos_;
const NUdf::ISecureParamsProvider* SecureParamsProvider_;
- TString ModuleIR_;
- TString ModuleIRUniqID_;
- TString IRFunctionName_;
+ TString ModuleIR_;
+ TString ModuleIRUniqID_;
+ TString IRFunctionName_;
};
class TTypeInfoHelper : public NUdf::ITypeInfoHelper
diff --git a/ydb/library/yql/minikql/mkql_type_ops.cpp b/ydb/library/yql/minikql/mkql_type_ops.cpp
index d559bb108c..96b3cba817 100644
--- a/ydb/library/yql/minikql/mkql_type_ops.cpp
+++ b/ydb/library/yql/minikql/mkql_type_ops.cpp
@@ -401,11 +401,11 @@ NUdf::TUnboxedValuePod ValueToString(NUdf::EDataSlot type, NUdf::TUnboxedValuePo
break;
case NUdf::EDataSlot::Float:
- out << ::FloatToString(value.Get<float>());
+ out << ::FloatToString(value.Get<float>());
break;
case NUdf::EDataSlot::Double:
- out << ::FloatToString(value.Get<double>());
+ out << ::FloatToString(value.Get<double>());
break;
case NUdf::EDataSlot::String:
diff --git a/ydb/library/yql/minikql/pack_num.cpp b/ydb/library/yql/minikql/pack_num.cpp
index a632185d70..b2b5cdec06 100644
--- a/ydb/library/yql/minikql/pack_num.cpp
+++ b/ydb/library/yql/minikql/pack_num.cpp
@@ -1 +1 @@
-#include "pack_num.h"
+#include "pack_num.h"
diff --git a/ydb/library/yql/minikql/pack_num.h b/ydb/library/yql/minikql/pack_num.h
index e48fb78cad..cce78faa68 100644
--- a/ydb/library/yql/minikql/pack_num.h
+++ b/ydb/library/yql/minikql/pack_num.h
@@ -1,198 +1,198 @@
-#pragma once
-
+#pragma once
+
#include <util/system/compiler.h>
-#include <util/system/defaults.h>
+#include <util/system/defaults.h>
#include <util/system/unaligned_mem.h>
+
+namespace NKikimr {
+
+constexpr size_t MAX_PACKED32_SIZE = 5;
+constexpr size_t MAX_PACKED64_SIZE = 9;
-namespace NKikimr {
-
-constexpr size_t MAX_PACKED32_SIZE = 5;
-constexpr size_t MAX_PACKED64_SIZE = 9;
-
-Y_FORCE_INLINE size_t GetPack32Length(ui32 value) {
- if (value < 0x80) {
- return 1;
- }
- else if (value < 0x4000) {
- return 2;
- }
- else if (value < 0x200000) {
- return 3;
- }
- else {
- return 5;
- }
-}
-
+Y_FORCE_INLINE size_t GetPack32Length(ui32 value) {
+ if (value < 0x80) {
+ return 1;
+ }
+ else if (value < 0x4000) {
+ return 2;
+ }
+ else if (value < 0x200000) {
+ return 3;
+ }
+ else {
+ return 5;
+ }
+}
+
[[nodiscard]] Y_FORCE_INLINE size_t Pack32(ui32 value, char* buffer) {
- if (value < 0x80) {
- buffer[0] = value << 0x1;
- return 1;
- }
- else if (value < 0x4000) {
+ if (value < 0x80) {
+ buffer[0] = value << 0x1;
+ return 1;
+ }
+ else if (value < 0x4000) {
WriteUnaligned<ui16>(((ui16*)(buffer)), ui16((value << 2) | 0x1));
- return 2;
- }
- else if (value < 0x200000) {
+ return 2;
+ }
+ else if (value < 0x200000) {
WriteUnaligned<ui32>(((ui32*)(buffer)), ui32((value << 3) | 0x3));
- return 3;
- }
- else {
- buffer[0] = 0x7;
+ return 3;
+ }
+ else {
+ buffer[0] = 0x7;
WriteUnaligned<ui32>((ui32*)(buffer + 1), ui32(value));
- return 5;
- }
-}
-
+ return 5;
+ }
+}
+
[[nodiscard]] Y_FORCE_INLINE size_t Unpack32(const char* buffer, size_t length, ui32& value) {
- if (Y_UNLIKELY(!length)) {
- return 0;
- }
-
- size_t count;
- if (0 == (buffer[0] & 0x1)) {
- count = 1;
- value = ((const ui8*)buffer)[0] >> 1;
+ if (Y_UNLIKELY(!length)) {
+ return 0;
+ }
+
+ size_t count;
+ if (0 == (buffer[0] & 0x1)) {
+ count = 1;
+ value = ((const ui8*)buffer)[0] >> 1;
} else if (0 == (buffer[0] & 0x2)) {
- count = 2;
- if (Y_UNLIKELY(length < count)) {
- return 0;
- }
+ count = 2;
+ if (Y_UNLIKELY(length < count)) {
+ return 0;
+ }
value = ReadUnaligned<ui16>((const ui16*)buffer) >> 2;
} else if (0 == (buffer[0] & 0x4)) {
- count = 3;
- if (Y_UNLIKELY(length < count)) {
- return 0;
- }
+ count = 3;
+ if (Y_UNLIKELY(length < count)) {
+ return 0;
+ }
value = (ReadUnaligned<ui16>((const ui16*)buffer) >> 3) | (((const ui8*)buffer)[2] << 13);
- } else {
- count = 5;
- if (Y_UNLIKELY(length < count || buffer[0] != 0x7)) {
- return 0;
- }
+ } else {
+ count = 5;
+ if (Y_UNLIKELY(length < count || buffer[0] != 0x7)) {
+ return 0;
+ }
value = ReadUnaligned<ui32>((const ui32*)(buffer + 1));
- }
-
- return count;
-}
-
-Y_FORCE_INLINE size_t GetPack64Length(ui64 value) {
- if (value < 0x80) {
- return 1;
- }
- else if (value < 0x4000) {
- return 2;
- }
- else if (value < 0x200000) {
- return 3;
- }
- else if (value < 0x10000000) {
- return 4;
- }
- else if (value < 0x800000000) {
- return 5;
- }
- else if (value < 0x40000000000) {
- return 6;
- }
- else if (value < 0x2000000000000) {
- return 7;
- }
- else {
- return 9;
- }
-}
+ }
+
+ return count;
+}
+
+Y_FORCE_INLINE size_t GetPack64Length(ui64 value) {
+ if (value < 0x80) {
+ return 1;
+ }
+ else if (value < 0x4000) {
+ return 2;
+ }
+ else if (value < 0x200000) {
+ return 3;
+ }
+ else if (value < 0x10000000) {
+ return 4;
+ }
+ else if (value < 0x800000000) {
+ return 5;
+ }
+ else if (value < 0x40000000000) {
+ return 6;
+ }
+ else if (value < 0x2000000000000) {
+ return 7;
+ }
+ else {
+ return 9;
+ }
+}
[[nodiscard]] Y_FORCE_INLINE size_t Pack64(ui64 value, char* buffer) {
- if (value < 0x80) {
- buffer[0] = value << 1;
- return 1;
- }
- else if (value < 0x4000) {
+ if (value < 0x80) {
+ buffer[0] = value << 1;
+ return 1;
+ }
+ else if (value < 0x4000) {
WriteUnaligned<ui16>((ui16*)(buffer), ui16((value << 2) | 0x1));
- return 2;
- }
- else if (value < 0x200000) {
+ return 2;
+ }
+ else if (value < 0x200000) {
WriteUnaligned<ui32>((ui32*)(buffer), ui32((value << 3) | 0x3));
- return 3;
- }
- else if (value < 0x10000000) {
+ return 3;
+ }
+ else if (value < 0x10000000) {
WriteUnaligned<ui32>((ui32*)(buffer), ui32((value << 4) | 0x7));
- return 4;
- }
- else if (value < 0x800000000) {
+ return 4;
+ }
+ else if (value < 0x800000000) {
WriteUnaligned<ui64>((ui64*)(buffer), ui64((value << 5) | 0xF));
- return 5;
- }
- else if (value < 0x40000000000) {
+ return 5;
+ }
+ else if (value < 0x40000000000) {
WriteUnaligned<ui64>((ui64*)(buffer), ui64((value << 6) | 0x1F));
- return 6;
- }
- else if (value < 0x2000000000000) {
+ return 6;
+ }
+ else if (value < 0x2000000000000) {
WriteUnaligned<ui64>((ui64*)(buffer), ui64((value << 7) | 0x3F));
- return 7;
- }
- else {
- buffer[0] = 0x7F;
+ return 7;
+ }
+ else {
+ buffer[0] = 0x7F;
WriteUnaligned<ui64>((ui64*)(buffer + 1), value);
- return 9;
- }
-}
-
+ return 9;
+ }
+}
+
[[nodiscard]] Y_FORCE_INLINE size_t Unpack64(const char* buffer, size_t length, ui64& value) {
- if (Y_UNLIKELY(!length)) {
- return 0;
- }
-
- size_t count;
- if (0 == (buffer[0] & 0x1)) {
- count = 1;
- value = ((const ui8*)buffer)[0] >> 1;
+ if (Y_UNLIKELY(!length)) {
+ return 0;
+ }
+
+ size_t count;
+ if (0 == (buffer[0] & 0x1)) {
+ count = 1;
+ value = ((const ui8*)buffer)[0] >> 1;
} else if (0 == (buffer[0] & 0x2)) {
- count = 2;
- if (Y_UNLIKELY(length < count)) {
- return 0;
- }
+ count = 2;
+ if (Y_UNLIKELY(length < count)) {
+ return 0;
+ }
value = ReadUnaligned<ui16>((const ui16*)buffer) >> 2;
} else if (0 == (buffer[0] & 0x4)) {
- count = 3;
- if (Y_UNLIKELY(length < count)) {
- return 0;
- }
+ count = 3;
+ if (Y_UNLIKELY(length < count)) {
+ return 0;
+ }
value = (ui64(ReadUnaligned<ui16>((const ui16*)buffer)) >> 3) | (ui64(((const ui8*)buffer)[2]) << 13);
} else if (0 == (buffer[0] & 0x8)) {
- count = 4;
- if (Y_UNLIKELY(length < count)) {
- return 0;
- }
+ count = 4;
+ if (Y_UNLIKELY(length < count)) {
+ return 0;
+ }
value = ReadUnaligned<ui32>((const ui32*)buffer) >> 4;
} else if (0 == (buffer[0] & 0x10)) {
- count = 5;
- if (Y_UNLIKELY(length < count)) {
- return 0;
- }
+ count = 5;
+ if (Y_UNLIKELY(length < count)) {
+ return 0;
+ }
value = (ui64(ReadUnaligned<ui32>((const ui32*)buffer)) >> 5) | (ui64(((const ui8*)buffer)[4]) << 27);
} else if (0 == (buffer[0] & 0x20)) {
- count = 6;
- if (Y_UNLIKELY(length < count)) {
- return 0;
- }
+ count = 6;
+ if (Y_UNLIKELY(length < count)) {
+ return 0;
+ }
value = (ui64(ReadUnaligned<ui32>((const ui32*)buffer)) >> 6) | (ui64(ReadUnaligned<ui16>(((const ui16*)buffer) + 2)) << 26);
} else if (0 == (buffer[0] & 0x40)) {
- count = 7;
- if (Y_UNLIKELY(length < count)) {
- return 0;
- }
+ count = 7;
+ if (Y_UNLIKELY(length < count)) {
+ return 0;
+ }
value = (ui64(ReadUnaligned<ui32>((const ui32*)buffer)) >> 7) | (ui64(ReadUnaligned<ui16>(((const ui16*)buffer) + 2)) << 25) | (ui64(((const ui8*)buffer)[6]) << 41);
- } else {
- count = 9;
- if (Y_UNLIKELY(length < count || buffer[0] != 0x7F)) {
- return 0;
- }
+ } else {
+ count = 9;
+ if (Y_UNLIKELY(length < count || buffer[0] != 0x7F)) {
+ return 0;
+ }
value = ReadUnaligned<ui64>((const ui64*)(buffer + 1));
- }
-
- return count;
-}
-
-} // NKikimr
+ }
+
+ return count;
+}
+
+} // NKikimr
diff --git a/ydb/library/yql/minikql/pack_num_ut.cpp b/ydb/library/yql/minikql/pack_num_ut.cpp
index bfc9c79309..3d78d1a5ef 100644
--- a/ydb/library/yql/minikql/pack_num_ut.cpp
+++ b/ydb/library/yql/minikql/pack_num_ut.cpp
@@ -1,69 +1,69 @@
#include <ydb/library/yql/minikql/pack_num.h>
-
+
#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/generic/ylimits.h>
-
+
+#include <util/generic/ylimits.h>
+
Y_UNIT_TEST_SUITE(TPackNumTest) {
- static ui32 TEST_DATA_UI32[] = {
- 0,
- 0x80,
- 0x4000,
- 0x8000,
- 0x200000,
- 0x800000,
- 0x80000000,
- Max<ui32>()
- };
- static ui64 TEST_DATA_UI64[] = {
- 0ull,
- 0x80ull,
- 0x4000ull,
- 0x8000ull,
- 0x200000ull,
- 0x800000ull,
- 0x10000000ull,
- 0x80000000ull,
- 0x800000000ull,
- 0x8000000000ull,
- 0x40000000000ull,
- 0x800000000000ull,
- 0x2000000000000ull,
- 0x80000000000000ull,
- 0x8000000000000000ull,
- Max<ui64>()};
-
- static size_t DELTA = 10;
-
+ static ui32 TEST_DATA_UI32[] = {
+ 0,
+ 0x80,
+ 0x4000,
+ 0x8000,
+ 0x200000,
+ 0x800000,
+ 0x80000000,
+ Max<ui32>()
+ };
+ static ui64 TEST_DATA_UI64[] = {
+ 0ull,
+ 0x80ull,
+ 0x4000ull,
+ 0x8000ull,
+ 0x200000ull,
+ 0x800000ull,
+ 0x10000000ull,
+ 0x80000000ull,
+ 0x800000000ull,
+ 0x8000000000ull,
+ 0x40000000000ull,
+ 0x800000000000ull,
+ 0x2000000000000ull,
+ 0x80000000000000ull,
+ 0x8000000000000000ull,
+ Max<ui64>()};
+
+ static size_t DELTA = 10;
+
Y_UNIT_TEST(PackUnpack32) {
- char buf[10];
- for (ui32 num: TEST_DATA_UI32) {
- for (ui32 i = num - Min<ui32>(num, DELTA); i < num + Min<ui32>(DELTA, Max<ui32>() - num); ++i) {
- size_t packedLen = NKikimr::Pack32(i, buf);
- UNIT_ASSERT(packedLen > 0);
- UNIT_ASSERT(packedLen <= 5);
- UNIT_ASSERT_VALUES_EQUAL(packedLen, NKikimr::GetPack32Length(i));
- ui32 unpacked = 0;
- size_t unpackedLen = NKikimr::Unpack32(buf, packedLen, unpacked);
- UNIT_ASSERT_VALUES_EQUAL(i, unpacked);
- UNIT_ASSERT_VALUES_EQUAL(packedLen, unpackedLen);
- }
- }
- }
-
+ char buf[10];
+ for (ui32 num: TEST_DATA_UI32) {
+ for (ui32 i = num - Min<ui32>(num, DELTA); i < num + Min<ui32>(DELTA, Max<ui32>() - num); ++i) {
+ size_t packedLen = NKikimr::Pack32(i, buf);
+ UNIT_ASSERT(packedLen > 0);
+ UNIT_ASSERT(packedLen <= 5);
+ UNIT_ASSERT_VALUES_EQUAL(packedLen, NKikimr::GetPack32Length(i));
+ ui32 unpacked = 0;
+ size_t unpackedLen = NKikimr::Unpack32(buf, packedLen, unpacked);
+ UNIT_ASSERT_VALUES_EQUAL(i, unpacked);
+ UNIT_ASSERT_VALUES_EQUAL(packedLen, unpackedLen);
+ }
+ }
+ }
+
Y_UNIT_TEST(PackUnpack64) {
- char buf[10];
- for (ui64 num: TEST_DATA_UI64) {
- for (ui64 i = num - Min<ui64>(num, DELTA); i < num + Min<ui64>(DELTA, Max<ui64>() - num); ++i) {
- size_t packedLen = NKikimr::Pack64(i, buf);
- UNIT_ASSERT(packedLen > 0);
- UNIT_ASSERT(packedLen <= 9);
- UNIT_ASSERT_VALUES_EQUAL(packedLen, NKikimr::GetPack64Length(i));
- ui64 unpacked = 0;
- size_t unpackedLen = NKikimr::Unpack64(buf, packedLen, unpacked);
- UNIT_ASSERT_VALUES_EQUAL(i, unpacked);
- UNIT_ASSERT_VALUES_EQUAL(packedLen, unpackedLen);
- }
- }
- }
-}
+ char buf[10];
+ for (ui64 num: TEST_DATA_UI64) {
+ for (ui64 i = num - Min<ui64>(num, DELTA); i < num + Min<ui64>(DELTA, Max<ui64>() - num); ++i) {
+ size_t packedLen = NKikimr::Pack64(i, buf);
+ UNIT_ASSERT(packedLen > 0);
+ UNIT_ASSERT(packedLen <= 9);
+ UNIT_ASSERT_VALUES_EQUAL(packedLen, NKikimr::GetPack64Length(i));
+ ui64 unpacked = 0;
+ size_t unpackedLen = NKikimr::Unpack64(buf, packedLen, unpacked);
+ UNIT_ASSERT_VALUES_EQUAL(i, unpacked);
+ UNIT_ASSERT_VALUES_EQUAL(packedLen, unpackedLen);
+ }
+ }
+ }
+}
diff --git a/ydb/library/yql/minikql/perf/packer/packer.cpp b/ydb/library/yql/minikql/perf/packer/packer.cpp
index f6222a95c0..c45ae10539 100644
--- a/ydb/library/yql/minikql/perf/packer/packer.cpp
+++ b/ydb/library/yql/minikql/perf/packer/packer.cpp
@@ -5,53 +5,53 @@
#include <ydb/library/yql/minikql/invoke_builtins/mkql_builtins.h>
#include <ydb/library/yql/minikql/mkql_program_builder.h>
#include <ydb/library/yql/minikql/mkql_function_registry.h>
-
-#include <util/datetime/cputimer.h>
-
-using namespace NKikimr;
-using namespace NKikimr::NMiniKQL;
-using namespace NKikimr::NUdf;
-
-static const size_t LIST_SIZE = 5000000ul;
-
-int main(int, char**) {
-
- auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
-
- TScopedAlloc alloc;
- TTypeEnvironment env(alloc);
- TMemoryUsageInfo memInfo("bench");
- THolderFactory holderFactory(alloc.Ref(), memInfo, functionRegistry.Get());
- TProgramBuilder pgmBuilder(env, *functionRegistry);
-
- auto listType = pgmBuilder.NewListType(pgmBuilder.NewTupleType({
- pgmBuilder.NewOptionalType(pgmBuilder.NewDataType(EDataSlot::Uint64)),
- pgmBuilder.NewDataType(EDataSlot::Uint64)
- }));
- TUnboxedValueVector listItems;
- for (ui64 i = 0; i < LIST_SIZE; ++i) {
- TUnboxedValueVector tupleItems;
- if (i % 2 == 0) {
- tupleItems.emplace_back();
- } else {
- tupleItems.emplace_back(TUnboxedValuePod(i));
- }
- tupleItems.emplace_back(TUnboxedValuePod(i));
-
- listItems.emplace_back(holderFactory.VectorAsArray(tupleItems));
- }
-
- TUnboxedValue list(holderFactory.VectorAsArray(listItems));
-
- TValuePacker packer(true, listType);
-
- TSimpleTimer timer;
- TStringBuf packed = packer.Pack(list);
- Cerr << "[pack] Elapsed: " << timer.Get() << "\n";
-
- timer.Reset();
- auto unpackedList = packer.Unpack(packed, holderFactory);
- Cerr << "[unpack] Elapsed: " << timer.Get() << "\n";
-
- return 0;
-}
+
+#include <util/datetime/cputimer.h>
+
+using namespace NKikimr;
+using namespace NKikimr::NMiniKQL;
+using namespace NKikimr::NUdf;
+
+static const size_t LIST_SIZE = 5000000ul;
+
+int main(int, char**) {
+
+ auto functionRegistry = CreateFunctionRegistry(CreateBuiltinRegistry());
+
+ TScopedAlloc alloc;
+ TTypeEnvironment env(alloc);
+ TMemoryUsageInfo memInfo("bench");
+ THolderFactory holderFactory(alloc.Ref(), memInfo, functionRegistry.Get());
+ TProgramBuilder pgmBuilder(env, *functionRegistry);
+
+ auto listType = pgmBuilder.NewListType(pgmBuilder.NewTupleType({
+ pgmBuilder.NewOptionalType(pgmBuilder.NewDataType(EDataSlot::Uint64)),
+ pgmBuilder.NewDataType(EDataSlot::Uint64)
+ }));
+ TUnboxedValueVector listItems;
+ for (ui64 i = 0; i < LIST_SIZE; ++i) {
+ TUnboxedValueVector tupleItems;
+ if (i % 2 == 0) {
+ tupleItems.emplace_back();
+ } else {
+ tupleItems.emplace_back(TUnboxedValuePod(i));
+ }
+ tupleItems.emplace_back(TUnboxedValuePod(i));
+
+ listItems.emplace_back(holderFactory.VectorAsArray(tupleItems));
+ }
+
+ TUnboxedValue list(holderFactory.VectorAsArray(listItems));
+
+ TValuePacker packer(true, listType);
+
+ TSimpleTimer timer;
+ TStringBuf packed = packer.Pack(list);
+ Cerr << "[pack] Elapsed: " << timer.Get() << "\n";
+
+ timer.Reset();
+ auto unpackedList = packer.Unpack(packed, holderFactory);
+ Cerr << "[unpack] Elapsed: " << timer.Get() << "\n";
+
+ return 0;
+}
diff --git a/ydb/library/yql/minikql/perf/packer/ya.make b/ydb/library/yql/minikql/perf/packer/ya.make
index cb7b6e7d10..2b5f69823f 100644
--- a/ydb/library/yql/minikql/perf/packer/ya.make
+++ b/ydb/library/yql/minikql/perf/packer/ya.make
@@ -1,25 +1,25 @@
-PROGRAM()
-
-ALLOCATOR(J)
-
-OWNER(
- udovichenko-r
- g:yql
-)
-
-PEERDIR(
+PROGRAM()
+
+ALLOCATOR(J)
+
+OWNER(
+ udovichenko-r
+ g:yql
+)
+
+PEERDIR(
ydb/library/yql/minikql
ydb/library/yql/minikql/comp_nodes
ydb/library/yql/minikql/computation
ydb/library/yql/minikql/invoke_builtins
ydb/library/yql/public/udf
ydb/library/yql/public/udf/service/exception_policy
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-SRCS(
- packer.cpp
-)
-
-END()
+SRCS(
+ packer.cpp
+)
+
+END()
diff --git a/ydb/library/yql/minikql/perf/presort/presort.cpp b/ydb/library/yql/minikql/perf/presort/presort.cpp
index cfe4ddfb8f..6f06dc5120 100644
--- a/ydb/library/yql/minikql/perf/presort/presort.cpp
+++ b/ydb/library/yql/minikql/perf/presort/presort.cpp
@@ -5,7 +5,7 @@
#include <ydb/library/yql/minikql/mkql_string_util.h>
#include <ydb/library/yql/public/udf/udf_types.h>
-#include <library/cpp/presort/presort.h>
+#include <library/cpp/presort/presort.h>
#include <util/random/random.h>
#include <util/datetime/cputimer.h>
diff --git a/ydb/library/yql/minikql/primes.cpp b/ydb/library/yql/minikql/primes.cpp
index 334b882844..54504b54b5 100644
--- a/ydb/library/yql/minikql/primes.cpp
+++ b/ydb/library/yql/minikql/primes.cpp
@@ -1,20 +1,20 @@
-#include "primes.h"
-
-#include <util/generic/array_size.h>
-#include <util/generic/algorithm.h>
-
-namespace NKikimr {
-
-static const unsigned long PRIMES[] = {
-#include "primes.gen"
-};
-
-unsigned long FindNearestPrime(unsigned long num) {
- if (num <= *PRIMES) {
- return *PRIMES;
- }
-
- return *LowerBound(PRIMES, PRIMES + Y_ARRAY_SIZE(PRIMES) - 1, num);
-}
-
-}
+#include "primes.h"
+
+#include <util/generic/array_size.h>
+#include <util/generic/algorithm.h>
+
+namespace NKikimr {
+
+static const unsigned long PRIMES[] = {
+#include "primes.gen"
+};
+
+unsigned long FindNearestPrime(unsigned long num) {
+ if (num <= *PRIMES) {
+ return *PRIMES;
+ }
+
+ return *LowerBound(PRIMES, PRIMES + Y_ARRAY_SIZE(PRIMES) - 1, num);
+}
+
+}
diff --git a/ydb/library/yql/minikql/primes.gen b/ydb/library/yql/minikql/primes.gen
index cc1a553784..e1d4d55b2a 100644
--- a/ydb/library/yql/minikql/primes.gen
+++ b/ydb/library/yql/minikql/primes.gen
@@ -1,51 +1,51 @@
-7ul,
-11ul,
-17ul,
-29ul,
-43ul,
-67ul,
-101ul,
-151ul,
-227ul,
-347ul,
-521ul,
-787ul,
-1181ul,
-1777ul,
-2671ul,
-4007ul,
-6011ul,
-9029ul,
-13553ul,
-20333ul,
-30509ul,
-45763ul,
-68659ul,
-103001ul,
-154501ul,
-231779ul,
-347671ul,
-521519ul,
-782297ul,
-1173463ul,
-1760203ul,
-2640317ul,
-3960497ul,
-5940761ul,
-8911141ul,
-13366711ul,
-20050081ul,
-30075127ul,
-45112693ul,
-67669079ul,
-101503627ul,
-152255461ul,
-228383273ul,
-342574909ul,
-513862367ul,
-770793589ul,
-1156190419ul,
-1734285653ul,
-2601428513ul,
-3902142817ul,
-4294967291ul, \ No newline at end of file
+7ul,
+11ul,
+17ul,
+29ul,
+43ul,
+67ul,
+101ul,
+151ul,
+227ul,
+347ul,
+521ul,
+787ul,
+1181ul,
+1777ul,
+2671ul,
+4007ul,
+6011ul,
+9029ul,
+13553ul,
+20333ul,
+30509ul,
+45763ul,
+68659ul,
+103001ul,
+154501ul,
+231779ul,
+347671ul,
+521519ul,
+782297ul,
+1173463ul,
+1760203ul,
+2640317ul,
+3960497ul,
+5940761ul,
+8911141ul,
+13366711ul,
+20050081ul,
+30075127ul,
+45112693ul,
+67669079ul,
+101503627ul,
+152255461ul,
+228383273ul,
+342574909ul,
+513862367ul,
+770793589ul,
+1156190419ul,
+1734285653ul,
+2601428513ul,
+3902142817ul,
+4294967291ul, \ No newline at end of file
diff --git a/ydb/library/yql/minikql/primes.h b/ydb/library/yql/minikql/primes.h
index 33b16a2575..ea166e5df6 100644
--- a/ydb/library/yql/minikql/primes.h
+++ b/ydb/library/yql/minikql/primes.h
@@ -1,7 +1,7 @@
-#pragma once
-
-namespace NKikimr {
-
-unsigned long FindNearestPrime(unsigned long num);
-
-}
+#pragma once
+
+namespace NKikimr {
+
+unsigned long FindNearestPrime(unsigned long num);
+
+}
diff --git a/ydb/library/yql/minikql/ya.make b/ydb/library/yql/minikql/ya.make
index c4b608128d..adb4449bd0 100644
--- a/ydb/library/yql/minikql/ya.make
+++ b/ydb/library/yql/minikql/ya.make
@@ -13,7 +13,7 @@ SRCS(
compact_hash.cpp
compact_hash.h
defs.h
- mkql_alloc.cpp
+ mkql_alloc.cpp
mkql_function_metadata.h
mkql_function_registry.cpp
mkql_function_registry.h
diff --git a/ydb/library/yql/mount/lib/yql/core.yql b/ydb/library/yql/mount/lib/yql/core.yql
index f12c459358..9dd30b96ee 100644
--- a/ydb/library/yql/mount/lib/yql/core.yql
+++ b/ydb/library/yql/mount/lib/yql/core.yql
@@ -118,18 +118,18 @@ def signature(script, name):
raise RuntimeError("Failed to extract type signature in the function: " + name + ", reason: \n" + out.getvalue())
@@))
-(let PythonFuncSignature (lambda '(flavor script name) (Apply
+(let PythonFuncSignature (lambda '(flavor script name) (Apply
(ScriptUdf flavor 'signature (ParseType '"(String, String)->String") PythonFuncSignatureScript) script name)))
(let KeysImpl (lambda '(seq)
- (MatchType seq 'Dict (lambda '() (DictKeys seq)) (lambda '()
+ (MatchType seq 'Dict (lambda '() (DictKeys seq)) (lambda '()
(MatchType seq 'EmptyDict (lambda '() (EmptyList)) (lambda '() seq)))
)))
(let ToSet (lambda '(list) (ToDict list (lambda '(x) x) (lambda '(x) (Void)) '('Hashed 'One))))
(let SetIsDisjointImpl (lambda '(dict scan) (Not (HasItems (SkipWhile
(Apply KeysImpl scan)
- (lambda '(item) (Not (Contains dict item))))))))
+ (lambda '(item) (Not (Contains dict item))))))))
(let AutoMap2 (lambda '(f) (lambda '(x y) (MatchType x 'Optional
(lambda '() (MatchType y 'Optional
@@ -162,10 +162,10 @@ def signature(script, name):
(let SetIncludesImpl (lambda '(dict scan) (Not (HasItems (SkipWhile
(Apply KeysImpl scan)
- (lambda '(item) (Contains dict item)))))))
+ (lambda '(item) (Contains dict item)))))))
(let SetDifferenceImpl (lambda '(x y) (ToDict (Filter (DictItems x) (lambda '(p)
- (Not (Contains y (Nth p '0)))))
+ (Not (Contains y (Nth p '0)))))
(lambda '(z) (Nth z '0)) (lambda '(z) (Nth z '1)) '('Hashed 'One) )))
(let SetUnionImpl (lambda '(f)
@@ -212,7 +212,7 @@ def signature(script, name):
(let SetSymmetricDifference3 (lambda '(x y f) (Apply (Apply AutoMap2 (Apply SetSymmetricDifferenceImpl f)) x y)))
(let SetSymmetricDifference2 (lambda '(x y) (Apply SetSymmetricDifference3 x y (lambda '(k a b) (Void)))))
-(let ListAggregateImpl (lambda '(list factory) (MatchType list 'EmptyList
+(let ListAggregateImpl (lambda '(list factory) (MatchType list 'EmptyList
(lambda '() (AsList (Null)))
(lambda '() (Extract (Aggregate
(Map list (lambda '(x) (AsStruct '('Data x))))
@@ -235,7 +235,7 @@ def signature(script, name):
'('Key)
'('('Data (Apply factory (StructType '('Data (ListItemType (DictPayloadType (TypeOf dict))))) (lambda '(x) (Member x 'Data)))))
'('('compact))
-)
+)
(lambda '(x) (Member x 'Key)) (lambda '(x) (Member x 'Data)) '('Hashed 'One)
)))))
@@ -272,9 +272,9 @@ def signature(script, name):
(lambda '(x) (NamedApply init '(x) (AsStruct) (DependsOn parent)))
(lambda '(x y) (NamedApply update '(x y) (AsStruct) (DependsOn parent)))
)))
- (let list_update (lambda '(value state parent) (IfPresent state (lambda '(unpackedState)
+ (let list_update (lambda '(value state parent) (IfPresent state (lambda '(unpackedState)
(Just (Fold (Apply extractor value) unpackedState
- (lambda '(x y) (NamedApply update '(x y) (AsStruct) (DependsOn parent))))))
+ (lambda '(x y) (NamedApply update '(x y) (AsStruct) (DependsOn parent))))))
(Fold1 (Apply extractor value)
(lambda '(x) (NamedApply init '(x) (AsStruct) (DependsOn parent)))
(lambda '(x y) (NamedApply update '(x y) (AsStruct) (DependsOn parent))) ))))
@@ -286,7 +286,7 @@ def signature(script, name):
(return (AggregationTraits type list_init list_update list_save list_load list_merge list_finish (Null)))
)))))
-(let ChooseMembersImpl (lambda '(str fields)
+(let ChooseMembersImpl (lambda '(str fields)
(Apply (EvaluateCode (LambdaCode (lambda '(row) (block '(
(let items (ListMap fields (lambda '(f) (block '(
(let atom (AtomCode f))
@@ -326,7 +326,7 @@ def signature(script, name):
))))) str))))
))
-(let DispatchStructFunc (lambda '(impl) (lambda '(str fields)
+(let DispatchStructFunc (lambda '(impl) (lambda '(str fields)
(MatchType str 'Null (lambda '() (Null)) (lambda '()
(MatchType str 'Optional (lambda '() (Map str (lambda '(unpacked) (Apply impl unpacked fields))))
(lambda '() (Apply impl str fields))))
@@ -336,12 +336,12 @@ def signature(script, name):
(let RemoveMembers (Apply DispatchStructFunc RemoveMembersImpl))
(let ForceRemoveMembers (Apply DispatchStructFunc ForceRemoveMembersImpl))
-(let StructMembers (lambda '(str)
+(let StructMembers (lambda '(str)
(MatchType str 'Null (lambda '() (List (ListType (DataType 'String)))) (lambda '()
(Extract (StructTypeComponents (TypeHandle (Apply RemoveOptionalType (TypeOf str)))) 'Name))
)))
-(let GatherMembersImpl (lambda '(str)
+(let GatherMembersImpl (lambda '(str)
(Apply (EvaluateCode (LambdaCode (lambda '(strCode) (block '(
(let type (Apply RemoveOptionalType (TypeOf str)))
(let fields (ListExtract (StructTypeComponents (TypeHandle type)) 'Name))
@@ -368,32 +368,32 @@ def signature(script, name):
(let renameSet (Apply ToSet (ListExtract renames '0)))
(let oldItems (ListNotNull (ListMap originalFields (lambda '(f) (block '(
(let atom (AtomCode f))
- (return (If (Contains renameSet f) (Null)
+ (return (If (Contains renameSet f) (Null)
(ListCode atom (FuncCode (String 'Member) strCode atom))))
))))))
(let newItems (ListNotNull (ListMap renames (lambda '(r) (block '(
(let prevAtom (AtomCode (Nth r '0)))
(let newAtom (AtomCode (Nth r '1)))
(let item (ListCode newAtom (FuncCode (String 'Member) strCode prevAtom)))
- (return (If force (If (Contains originalSet (Nth r '0)) item (Null)) (Just item)))
+ (return (If force (If (Contains originalSet (Nth r '0)) item (Null)) (Just item)))
))))))
(return (MatchType type 'Null (lambda '() (ReprCode (Null))) (lambda '() (FuncCode (String 'AsStruct) (Extend oldItems newItems)))))
))))) str)
))
-(let RenameMembers (lambda '(str renames)
+(let RenameMembers (lambda '(str renames)
(MatchType str 'Null (lambda '() (Null)) (lambda '()
(MatchType str 'Optional (lambda '() (Map str (lambda '(unpacked) (Apply RenameMembersImpl unpacked renames (Bool '0)))))
(lambda '() (Apply RenameMembersImpl str renames (Bool '0)))))
)))
-(let ForceRenameMembers (lambda '(str renames)
+(let ForceRenameMembers (lambda '(str renames)
(MatchType str 'Null (lambda '() (Null)) (lambda '()
(MatchType str 'Optional (lambda '() (Map str (lambda '(unpacked) (Apply RenameMembersImpl unpacked renames (Bool '1)))))
(lambda '() (Apply RenameMembersImpl str renames (Bool '1)))))
)))
-(let RenameMembers (lambda '(str renames)
+(let RenameMembers (lambda '(str renames)
(MatchType str 'Null (lambda '() (Null)) (lambda '()
(MatchType str 'Optional (lambda '() (Map str (lambda '(unpacked) (Apply RenameMembersImpl unpacked renames (Bool '0)))))
(lambda '() (Apply RenameMembersImpl str renames (Bool '0)))))
@@ -401,13 +401,13 @@ def signature(script, name):
(let SpreadMembersImpl (lambda '(list fields force) (block '(
(let fieldsSet (Apply ToSet fields))
- (let check (lambda '(item) (Ensure item (SqlIn fieldsSet (Nth item '0) '())
+ (let check (lambda '(item) (Ensure item (SqlIn fieldsSet (Nth item '0) '())
(Concat (String '"Member not found: ") (Nth item '0)))))
(let list (If force list (ListMap list check)))
(let type (Apply RemoveOptionalType (TypeOf list)))
(let itemType (TupleElementType (ListItemType type) '1))
(let optType (OptionalType (Apply RemoveOptionalType itemType)))
- (let empty (MatchType type 'EmptyList (lambda '() (Null)) (lambda '()
+ (let empty (MatchType type 'EmptyList (lambda '() (Null)) (lambda '()
(MatchType type 'Null (lambda '() (Null)) (lambda '() (Nothing optType)))))
)
(return (Apply (EvaluateCode (LambdaCode (lambda '(listCode) (block '(
@@ -430,13 +430,13 @@ def signature(script, name):
))))) list))
))))
-(let SpreadMembers (lambda '(list fields)
+(let SpreadMembers (lambda '(list fields)
(MatchType list 'Null (lambda '() (Null)) (lambda '()
(MatchType list 'Optional (lambda '() (Map list (lambda '(unpacked) (Apply SpreadMembersImpl unpacked fields (Bool '0)))))
(lambda '() (Apply SpreadMembersImpl list fields (Bool '0)))))
)))
-(let ForceSpreadMembers (lambda '(list fields)
+(let ForceSpreadMembers (lambda '(list fields)
(MatchType list 'Null (lambda '() (Null)) (lambda '()
(MatchType list 'Optional (lambda '() (Map list (lambda '(unpacked) (Apply SpreadMembersImpl unpacked fields (Bool '1)))))
(lambda '() (Apply SpreadMembersImpl list fields (Bool '1)))))
diff --git a/ydb/library/yql/providers/clickhouse/provider/ya.make b/ydb/library/yql/providers/clickhouse/provider/ya.make
index 05effd38eb..df99a4474e 100644
--- a/ydb/library/yql/providers/clickhouse/provider/ya.make
+++ b/ydb/library/yql/providers/clickhouse/provider/ya.make
@@ -8,11 +8,11 @@ SRCS(
yql_clickhouse_datasink_type_ann.cpp
yql_clickhouse_datasource.cpp
yql_clickhouse_datasource_type_ann.cpp
- yql_clickhouse_dq_integration.cpp
+ yql_clickhouse_dq_integration.cpp
yql_clickhouse_io_discovery.cpp
yql_clickhouse_load_meta.cpp
yql_clickhouse_logical_opt.cpp
- yql_clickhouse_mkql_compiler.cpp
+ yql_clickhouse_mkql_compiler.cpp
yql_clickhouse_physical_opt.cpp
yql_clickhouse_provider.cpp
yql_clickhouse_provider.h
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp
index e23600f80f..6cbd36d4a3 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_datasource.cpp
@@ -1,5 +1,5 @@
#include "yql_clickhouse_provider_impl.h"
-#include "yql_clickhouse_dq_integration.h"
+#include "yql_clickhouse_dq_integration.h"
#include <ydb/library/yql/providers/clickhouse/expr_nodes/yql_clickhouse_expr_nodes.h>
#include <ydb/library/yql/providers/common/provider/yql_provider.h>
@@ -15,7 +15,7 @@ using namespace NNodes;
namespace {
-class TClickHouseDataSource : public TDataProviderBase {
+class TClickHouseDataSource : public TDataProviderBase {
public:
TClickHouseDataSource(TClickHouseState::TPtr state, IHTTPGateway::TPtr gateway)
: State_(state)
@@ -103,11 +103,11 @@ public:
}
}
- IDqIntegration* GetDqIntegration() override {
- return DqIntegration_.Get();
- }
-
-
+ IDqIntegration* GetDqIntegration() override {
+ return DqIntegration_.Get();
+ }
+
+
private:
const TClickHouseState::TPtr State_;
const THolder<IGraphTransformer> IODiscoveryTransformer_;
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp
index a882da93a0..66a682e819 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.cpp
@@ -1,6 +1,6 @@
-#include "yql_clickhouse_dq_integration.h"
-#include "yql_clickhouse_mkql_compiler.h"
-
+#include "yql_clickhouse_dq_integration.h"
+#include "yql_clickhouse_mkql_compiler.h"
+
#include <ydb/library/yql/providers/common/dq/yql_dq_integration_impl.h>
#include <ydb/library/yql/providers/dq/common/yql_dq_settings.h>
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
@@ -9,26 +9,26 @@
#include <ydb/library/yql/providers/clickhouse/proto/range.pb.h>
#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
#include <ydb/library/yql/ast/yql_expr.h>
-
-namespace NYql {
-
-using namespace NNodes;
-
+
+namespace NYql {
+
+using namespace NNodes;
+
namespace {
-class TClickHouseDqIntegration: public TDqIntegrationBase {
-public:
+class TClickHouseDqIntegration: public TDqIntegrationBase {
+public:
TClickHouseDqIntegration(TClickHouseState::TPtr state)
- : State_(state)
+ : State_(state)
{}
-
+
TMaybe<ui64> CanRead(const TDqSettings&, const TExprNode& read, TExprContext&, bool) override {
if (TClReadTable::Match(&read)) {
- return 0ul; // TODO: return real size
- }
- return Nothing();
- }
-
+ return 0ul; // TODO: return real size
+ }
+ return Nothing();
+ }
+
TExprNode::TPtr WrapRead(const TDqSettings&, const TExprNode::TPtr& read, TExprContext& ctx) override {
if (const auto maybeClReadTable = TMaybeNode<TClReadTable>(read)) {
const auto clReadTable = maybeClReadTable.Cast();
@@ -57,10 +57,10 @@ public:
.RowType(ExpandType(clReadTable.Pos(), *rowType, ctx))
.DataSource(clReadTable.DataSource().Cast<TCoDataSource>())
.Done().Ptr();
- }
- return read;
- }
-
+ }
+ return read;
+ }
+
ui64 Partition(const TDqSettings&, size_t, const TExprNode&, TVector<TString>& partitions, TString*, TExprContext&, bool) override {
partitions.clear();
NCH::TRange range;
@@ -111,16 +111,16 @@ public:
void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) override {
RegisterDqClickHouseMkqlCompilers(compiler, State_);
- }
-
-private:
+ }
+
+private:
const TClickHouseState::TPtr State_;
-};
-
-}
-
+};
+
+}
+
THolder<IDqIntegration> CreateClickHouseDqIntegration(TClickHouseState::TPtr state) {
return MakeHolder<TClickHouseDqIntegration>(state);
-}
+}
}
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.h b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.h
index e23e896c75..60e418db5f 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.h
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_dq_integration.h
@@ -1,13 +1,13 @@
-#pragma once
-
-#include "yql_clickhouse_provider.h"
-
+#pragma once
+
+#include "yql_clickhouse_provider.h"
+
#include <ydb/library/yql/providers/dq/interface/yql_dq_integration.h>
-
-#include <util/generic/ptr.h>
-
-namespace NYql {
-
+
+#include <util/generic/ptr.h>
+
+namespace NYql {
+
THolder<IDqIntegration> CreateClickHouseDqIntegration(TClickHouseState::TPtr state);
-
-}
+
+}
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp
index eeaeffcffc..803b895eb5 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_load_meta.cpp
@@ -213,7 +213,7 @@ private:
const auto& cols = row.back().GetArray();
if (cols.empty()) {
- ctx.AddError(TIssue({}, TStringBuilder() << "Table " << cluster << '.' << table << " doesn't exist."));
+ ctx.AddError(TIssue({}, TStringBuilder() << "Table " << cluster << '.' << table << " doesn't exist."));
return {nullptr, {}};
}
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_logical_opt.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_logical_opt.cpp
index 7d3c31a45e..bc45d93494 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_logical_opt.cpp
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_logical_opt.cpp
@@ -10,7 +10,7 @@
#include <ydb/library/yql/core/yql_opt_utils.h>
#include <ydb/library/yql/utils/log/log.h>
-
+
namespace NYql {
using namespace NNodes;
@@ -26,7 +26,7 @@ public:
#define HNDL(name) "LogicalOptimizer-"#name, Hndl(&TClickHouseLogicalOptProposalTransformer::name)
AddHandler(0, &TCoLeft::Match, HNDL(TrimReadWorld));
AddHandler(0, &TCoExtractMembers::Match, HNDL(ExtractMembers));
- AddHandler(0, &TCoExtractMembers::Match, HNDL(ExtractMembersOverDqWrap));
+ AddHandler(0, &TCoExtractMembers::Match, HNDL(ExtractMembersOverDqWrap));
AddHandler(0, &TCoExtractMembers::Match, HNDL(ExtractMembersOverDqSourceWrap));
#undef HNDL
}
@@ -54,23 +54,23 @@ public:
.Done();
}
- TMaybeNode<TExprBase> ExtractMembersOverDqWrap(TExprBase node, TExprContext& ctx) const {
- auto extract = node.Cast<TCoExtractMembers>();
- auto input = extract.Input();
- auto read = input.Maybe<TDqReadWrap>().Input().Maybe<TClReadTable>();
- if (!read) {
- return node;
- }
-
- return Build<TDqReadWrap>(ctx, node.Pos())
- .InitFrom(input.Cast<TDqReadWrap>())
- .Input<TClReadTable>()
+ TMaybeNode<TExprBase> ExtractMembersOverDqWrap(TExprBase node, TExprContext& ctx) const {
+ auto extract = node.Cast<TCoExtractMembers>();
+ auto input = extract.Input();
+ auto read = input.Maybe<TDqReadWrap>().Input().Maybe<TClReadTable>();
+ if (!read) {
+ return node;
+ }
+
+ return Build<TDqReadWrap>(ctx, node.Pos())
+ .InitFrom(input.Cast<TDqReadWrap>())
+ .Input<TClReadTable>()
.InitFrom(read.Cast())
- .Columns(extract.Members())
- .Build()
- .Done();
- }
-
+ .Columns(extract.Members())
+ .Build()
+ .Done();
+ }
+
TMaybeNode<TExprBase> ExtractMembersOverDqSourceWrap(TExprBase node, TExprContext& ctx) const {
const auto extract = node.Cast<TCoExtractMembers>();
const auto input = extract.Input();
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_mkql_compiler.cpp b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_mkql_compiler.cpp
index 5a380898e8..84231efe92 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_mkql_compiler.cpp
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_mkql_compiler.cpp
@@ -1,21 +1,21 @@
-#include "yql_clickhouse_mkql_compiler.h"
-#include "yql_clickhouse_util.h"
-
+#include "yql_clickhouse_mkql_compiler.h"
+#include "yql_clickhouse_util.h"
+
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
#include <ydb/library/yql/providers/common/mkql/yql_type_mkql.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
#include <ydb/library/yql/minikql/mkql_node.h>
#include <ydb/library/yql/core/yql_opt_utils.h>
-
+
#include <library/cpp/json/json_writer.h>
-
-#include <algorithm>
-
-namespace NYql {
-
-using namespace NKikimr::NMiniKQL;
-using namespace NNodes;
-
+
+#include <algorithm>
+
+namespace NYql {
+
+using namespace NKikimr::NMiniKQL;
+using namespace NNodes;
+
namespace {
TRuntimeNode BuildNativeParseCall(TRuntimeNode input, TType* inputItemType, TType* outputItemType, const std::string_view& timezone, NCommon::TMkqlBuildContext& ctx)
@@ -33,37 +33,37 @@ TRuntimeNode BuildNativeParseCall(TRuntimeNode input, TType* inputItemType, TTyp
});
}
-TRuntimeNode BuildClickHouseInputCall(
- TType* outputType,
- TType* itemType,
- const TString& cluster,
- TStringBuf table,
- TStringBuf timezone,
- const TClickHouseState::TPtr& state,
+TRuntimeNode BuildClickHouseInputCall(
+ TType* outputType,
+ TType* itemType,
+ const TString& cluster,
+ TStringBuf table,
+ TStringBuf timezone,
+ const TClickHouseState::TPtr& state,
NCommon::TMkqlBuildContext& ctx)
-{
- TStringBuf db, dbTable;
- if (!table.TrySplit(".", db, dbTable)) {
- db = "default";
- dbTable = table;
- }
-
- auto structType = AS_TYPE(TStructType, itemType);
+{
+ TStringBuf db, dbTable;
+ if (!table.TrySplit(".", db, dbTable)) {
+ db = "default";
+ dbTable = table;
+ }
+
+ auto structType = AS_TYPE(TStructType, itemType);
auto endpoint = state->Configuration->Endpoints.FindPtr(cluster);
Y_ENSURE(endpoint);
-
+
const auto host = endpoint->first;
const auto colonPos = host.rfind(':');
YQL_ENSURE(colonPos != TString::npos, "Missing port: " << host);
-
+
const auto hostWithoutPort = host.substr(0, colonPos);
const auto port = FromString<ui16>(host.substr(colonPos + 1));
const bool secure = endpoint->second;
-
+
TString typeConfig;
TStringOutput stream(typeConfig);
NJson::TJsonWriter writer(&stream, NJson::TJsonWriterConfig());
-
+
writer.OpenMap();
writer.Write("db", db);
@@ -78,10 +78,10 @@ TRuntimeNode BuildClickHouseInputCall(
writer.Write(structType->GetMemberName(i));
}
writer.CloseArray();
-
+
writer.CloseMap();
writer.Flush();
-
+
const auto voidType = ctx.ProgramBuilder.NewVoid().GetStaticType();
TCallableTypeBuilder callbackTypeBuilder(ctx.ProgramBuilder.GetTypeEnvironment(), "", voidType);
const auto callbackType = callbackTypeBuilder.Build();
@@ -90,36 +90,36 @@ TRuntimeNode BuildClickHouseInputCall(
const auto argsType = ctx.ProgramBuilder.NewTupleType({ strType });
const auto userType = ctx.ProgramBuilder.NewTupleType({ argsType });
const auto retType = ctx.ProgramBuilder.NewStreamType(itemType);
-
- TCallableTypeBuilder funcTypeBuilder(ctx.ProgramBuilder.GetTypeEnvironment(), "UDF", retType);
- funcTypeBuilder.Add(strType);
+
+ TCallableTypeBuilder funcTypeBuilder(ctx.ProgramBuilder.GetTypeEnvironment(), "UDF", retType);
+ funcTypeBuilder.Add(strType);
const auto funcType = funcTypeBuilder.Build();
-
+
TCallableBuilder callbackBuilder(ctx.ProgramBuilder.GetTypeEnvironment(), "DqNotify", callbackType);
auto callback = TRuntimeNode(callbackBuilder.Build(), false);
- auto flow = ctx.ProgramBuilder.ToFlow(
- ctx.ProgramBuilder.Apply(
+ auto flow = ctx.ProgramBuilder.ToFlow(
+ ctx.ProgramBuilder.Apply(
ctx.ProgramBuilder.TypedUdf("ClickHouse.remoteSource", funcType,
ctx.ProgramBuilder.NewVoid(), userType, typeConfig),
TArrayRef<const TRuntimeNode>({ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(timezone)})
- ));
-
- if (!AS_TYPE(TFlowType, outputType)->GetItemType()->IsSameType(*itemType)) {
- flow = ctx.ProgramBuilder.ExpandMap(flow,
- [&](TRuntimeNode item) {
- TRuntimeNode::TList fields;
- fields.reserve(structType->GetMembersCount());
- auto i = 0U;
- std::generate_n(std::back_inserter(fields), structType->GetMembersCount(), [&](){ return ctx.ProgramBuilder.Member(item, structType->GetMemberName(i++)); });
- return fields;
- });
- }
-
- return flow;
-}
-
+ ));
+
+ if (!AS_TYPE(TFlowType, outputType)->GetItemType()->IsSameType(*itemType)) {
+ flow = ctx.ProgramBuilder.ExpandMap(flow,
+ [&](TRuntimeNode item) {
+ TRuntimeNode::TList fields;
+ fields.reserve(structType->GetMembersCount());
+ auto i = 0U;
+ std::generate_n(std::back_inserter(fields), structType->GetMembersCount(), [&](){ return ctx.ProgramBuilder.Member(item, structType->GetMemberName(i++)); });
+ return fields;
+ });
+ }
+
+ return flow;
+}
+
}
void RegisterDqClickHouseMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler, const TClickHouseState::TPtr& state) {
@@ -142,12 +142,12 @@ void RegisterDqClickHouseMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compi
const auto readType = clRead.Ref().GetTypeAnn()->Cast<TTupleExprType>()->GetItems().back();
const auto inputItemType = NCommon::BuildType(wrapper.Input().Ref(), *GetSeqItemType(readType), ctx.ProgramBuilder);
const auto cluster = clRead.DataSource().Cluster().StringValue();
- const auto outputType = NCommon::BuildType(wrapper.Ref(), *wrapper.Ref().GetTypeAnn(), ctx.ProgramBuilder);
+ const auto outputType = NCommon::BuildType(wrapper.Ref(), *wrapper.Ref().GetTypeAnn(), ctx.ProgramBuilder);
return BuildClickHouseInputCall(outputType, inputItemType, cluster, clRead.Table(), clRead.Timezone(), state, ctx);
- }
-
- return TRuntimeNode();
- });
-}
-
-}
+ }
+
+ return TRuntimeNode();
+ });
+}
+
+}
diff --git a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_mkql_compiler.h b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_mkql_compiler.h
index da2c2f1406..f14189c861 100644
--- a/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_mkql_compiler.h
+++ b/ydb/library/yql/providers/clickhouse/provider/yql_clickhouse_mkql_compiler.h
@@ -1,12 +1,12 @@
-#pragma once
-
-#include "yql_clickhouse_provider.h"
-
+#pragma once
+
+#include "yql_clickhouse_provider.h"
+
#include <ydb/library/yql/providers/common/mkql/yql_provider_mkql.h>
#include <ydb/library/yql/providers/clickhouse/expr_nodes/yql_clickhouse_expr_nodes.h>
-
-namespace NYql {
-
+
+namespace NYql {
+
void RegisterDqClickHouseMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler, const TClickHouseState::TPtr& state);
-
-}
+
+}
diff --git a/ydb/library/yql/providers/common/codec/ya.make b/ydb/library/yql/providers/common/codec/ya.make
index b3f3a92db5..54d0473a1a 100644
--- a/ydb/library/yql/providers/common/codec/ya.make
+++ b/ydb/library/yql/providers/common/codec/ya.make
@@ -11,8 +11,8 @@ SRCS(
yql_codec_results.h
yql_restricted_yson.cpp
yql_restricted_yson.h
- yql_codec_type_flags.cpp
- yql_codec_type_flags.h
+ yql_codec_type_flags.cpp
+ yql_codec_type_flags.h
)
PEERDIR(
@@ -23,8 +23,8 @@ PEERDIR(
library/cpp/yson
)
-YQL_LAST_ABI_VERSION()
+YQL_LAST_ABI_VERSION()
-GENERATE_ENUM_SERIALIZATION(yql_codec_type_flags.h)
+GENERATE_ENUM_SERIALIZATION(yql_codec_type_flags.h)
END()
diff --git a/ydb/library/yql/providers/common/codec/yql_codec.cpp b/ydb/library/yql/providers/common/codec/yql_codec.cpp
index 7587e188fa..2652dfb88a 100644
--- a/ydb/library/yql/providers/common/codec/yql_codec.cpp
+++ b/ydb/library/yql/providers/common/codec/yql_codec.cpp
@@ -1,7 +1,7 @@
#include "yql_codec.h"
#include "yql_restricted_yson.h"
-#include "yql_codec_type_flags.h"
-
+#include "yql_codec_type_flags.h"
+
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
#include <ydb/library/yql/public/decimal/yql_decimal.h>
@@ -19,10 +19,10 @@
#include <library/cpp/string_utils/base64/base64.h>
#include <library/cpp/yson/parser.h>
#include <library/cpp/yson/detail.h>
-
-#include <util/string/cast.h>
+
+#include <util/string/cast.h>
#include <util/generic/map.h>
-
+
namespace NYql {
namespace NCommon {
@@ -141,10 +141,10 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod&
}
for (ui32 i = 0, e = structType->GetMembersCount(); i < e; ++i) {
const ui32 pos = structPositions ? (*structPositions)[i] : i;
- if (pos < e) {
- writer.OnListItem();
+ if (pos < e) {
+ writer.OnListItem();
WriteYsonValueImpl(writer, value.GetElement(pos), structType->GetMemberType(pos), nullptr);
- }
+ }
}
writer.OnEndList();
@@ -170,7 +170,7 @@ void WriteYsonValueImpl(TYsonResultWriter& writer, const NUdf::TUnboxedValuePod&
} else {
writer.OnBeginList();
auto optionalType = AS_TYPE(TOptionalType, type);
- writer.OnListItem();
+ writer.OnListItem();
WriteYsonValueImpl(writer, value.GetOptionalValue(), optionalType->GetItemType(), nullptr);
writer.OnEndList();
}
@@ -265,51 +265,51 @@ TCodecContext::TCodecContext(
}
TMaybe<TVector<ui32>> CreateStructPositions(TType* inputType, const TVector<TString>* columns) {
- if (inputType->GetKind() != TType::EKind::Struct) {
- return Nothing();
- }
- auto inputStruct = AS_TYPE(TStructType, inputType);
+ if (inputType->GetKind() != TType::EKind::Struct) {
+ return Nothing();
+ }
+ auto inputStruct = AS_TYPE(TStructType, inputType);
TMap<TStringBuf, ui32> members;
TVector<ui32> structPositions(inputStruct->GetMembersCount(), Max<ui32>());
- for (ui32 i = 0; i < inputStruct->GetMembersCount(); ++i) {
- if (columns) {
- members.insert(std::make_pair(inputStruct->GetMemberName(i), i));
- } else {
- structPositions[i] = i;
- }
- }
- if (columns) {
- ui32 pos = 0;
- for (auto& column: *columns) {
- const ui32* idx = members.FindPtr(column);
- YQL_ENSURE(idx, "Unknown member: " << column);
- structPositions[pos] = *idx;
- ++pos;
- }
- }
- return structPositions;
-}
-
+ for (ui32 i = 0; i < inputStruct->GetMembersCount(); ++i) {
+ if (columns) {
+ members.insert(std::make_pair(inputStruct->GetMemberName(i), i));
+ } else {
+ structPositions[i] = i;
+ }
+ }
+ if (columns) {
+ ui32 pos = 0;
+ for (auto& column: *columns) {
+ const ui32* idx = members.FindPtr(column);
+ YQL_ENSURE(idx, "Unknown member: " << column);
+ structPositions[pos] = *idx;
+ ++pos;
+ }
+ }
+ return structPositions;
+}
+
namespace {
NYT::TNode DataValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NMiniKQL::TType* type) {
- YQL_ENSURE(type->GetKind() == TType::EKind::Data);
-
- auto dataType = AS_TYPE(TDataType, type);
- switch (dataType->GetSchemeType()) {
+ YQL_ENSURE(type->GetKind() == TType::EKind::Data);
+
+ auto dataType = AS_TYPE(TDataType, type);
+ switch (dataType->GetSchemeType()) {
case NUdf::TDataType<i32>::Id:
- return NYT::TNode(value.Get<i32>());
+ return NYT::TNode(value.Get<i32>());
case NUdf::TDataType<i64>::Id:
- return NYT::TNode(value.Get<i64>());
+ return NYT::TNode(value.Get<i64>());
case NUdf::TDataType<ui32>::Id:
- return NYT::TNode(value.Get<ui32>());
+ return NYT::TNode(value.Get<ui32>());
case NUdf::TDataType<ui64>::Id:
- return NYT::TNode(value.Get<ui64>());
+ return NYT::TNode(value.Get<ui64>());
case NUdf::TDataType<float>::Id:
- return NYT::TNode(value.Get<float>());
+ return NYT::TNode(value.Get<float>());
case NUdf::TDataType<double>::Id:
- return NYT::TNode(value.Get<double>());
+ return NYT::TNode(value.Get<double>());
case NUdf::TDataType<bool>::Id:
- return NYT::TNode(value.Get<bool>());
+ return NYT::TNode(value.Get<bool>());
case NUdf::TDataType<ui8>::Id:
return NYT::TNode((ui64)value.Get<ui8>());
case NUdf::TDataType<i8>::Id:
@@ -335,31 +335,31 @@ NYT::TNode DataValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr
return NYT::TNode(value.Get<i64>());
case NUdf::TDataType<NUdf::TTzDate>::Id: {
TStringStream out;
- out << value.Get<ui16>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId());
+ out << value.Get<ui16>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId());
return NYT::TNode(out.Str());
}
case NUdf::TDataType<NUdf::TTzDatetime>::Id: {
TStringStream out;
- out << value.Get<ui32>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId());
+ out << value.Get<ui32>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId());
return NYT::TNode(out.Str());
}
case NUdf::TDataType<NUdf::TTzTimestamp>::Id: {
TStringStream out;
- out << value.Get<ui64>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId());
+ out << value.Get<ui64>() << "," << NKikimr::NMiniKQL::GetTimezoneIANAName(value.GetTimezoneId());
return NYT::TNode(out.Str());
}
- case NUdf::TDataType<NUdf::TDecimal>::Id: {
- const auto params = static_cast<NKikimr::NMiniKQL::TDataDecimalType*>(type)->GetParams();
- return NYT::TNode(NDecimal::ToString(value.GetInt128(), params.first, params.second));
- }
+ case NUdf::TDataType<NUdf::TDecimal>::Id: {
+ const auto params = static_cast<NKikimr::NMiniKQL::TDataDecimalType*>(type)->GetParams();
+ return NYT::TNode(NDecimal::ToString(value.GetInt128(), params.first, params.second));
+ }
case NUdf::TDataType<NUdf::TJsonDocument>::Id: {
NUdf::TUnboxedValue json = ValueToString(EDataSlot::JsonDocument, value);
return NYT::TNode(ToString(TStringBuf(value.AsStringRef())));
}
- }
+ }
YQL_ENSURE(false, "Unsupported type: " << static_cast<int>(dataType->GetSchemeType()));
-}
-
+}
+
TExprNode::TPtr DataNodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode& type, const NYT::TNode& node, TExprContext& ctx) {
YQL_ENSURE(type.GetKind() == ETypeAnnotationKind::Data, "Expecting data type, got: " << type);
@@ -466,7 +466,7 @@ TString DataValueToString(const NKikimr::NUdf::TUnboxedValuePod& value, const TD
}
Y_FAIL("Unexpected");
-}
+}
} //namespace
NYT::TNode ValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NMiniKQL::TType* type) {
@@ -502,7 +502,7 @@ NYT::TNode ValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NM
}
return result;
}
-
+
TExprNode::TPtr NodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode& type, const NYT::TNode& node, TExprContext& ctx) {
TExprNode::TPtr result;
switch(type.GetKind()) {
@@ -747,8 +747,8 @@ NUdf::TUnboxedValue ReadYsonValue(TType* type,
cmd = buf.Read();
ui64 index = 0;
if (isTableFormat) {
- CHECK_EXPECTED(cmd, Uint64Marker);
- index = buf.ReadVarUI64();
+ CHECK_EXPECTED(cmd, Uint64Marker);
+ index = buf.ReadVarUI64();
} else {
index = ReadNextSerializedNumber<ui64>(cmd, buf);
}
@@ -1312,7 +1312,7 @@ extern "C" void ReadYsonContainerValue(TType* type, const NKikimr::NMiniKQL::THo
}
}
-NUdf::TUnboxedValue ReadSkiffData(TType* type, ui64 nativeYtTypeFlags, TInputBuf& buf) {
+NUdf::TUnboxedValue ReadSkiffData(TType* type, ui64 nativeYtTypeFlags, TInputBuf& buf) {
auto schemeType = static_cast<TDataType*>(type)->GetSchemeType();
switch (schemeType) {
case NUdf::TDataType<bool>::Id: {
@@ -1400,34 +1400,34 @@ NUdf::TUnboxedValue ReadSkiffData(TType* type, ui64 nativeYtTypeFlags, TInputBuf
}
case NUdf::TDataType<NUdf::TDecimal>::Id: {
- if (nativeYtTypeFlags & NTCF_DECIMAL) {
- auto const params = static_cast<TDataDecimalType*>(type)->GetParams();
- if (params.first < 10) {
- i32 data;
- buf.ReadMany((char*)&data, sizeof(data));
- return NUdf::TUnboxedValuePod(NDecimal::FromYtDecimal(data));
- } else if (params.first < 19) {
- i64 data;
- buf.ReadMany((char*)&data, sizeof(data));
- return NUdf::TUnboxedValuePod(NDecimal::FromYtDecimal(data));
- } else {
- YQL_ENSURE(params.first < 36);
- NDecimal::TInt128 data;
- buf.ReadMany((char*)&data, sizeof(data));
- return NUdf::TUnboxedValuePod(NDecimal::FromYtDecimal(data));
- }
- } else {
- ui32 size;
- buf.ReadMany(reinterpret_cast<char*>(&size), sizeof(size));
- const auto maxSize = sizeof(NDecimal::TInt128);
- YQL_ENSURE(size > 0U && size <= maxSize, "Bad decimal field size: " << size);
- char data[maxSize];
- buf.ReadMany(data, size);
- const auto& v = NDecimal::Deserialize(data);
- YQL_ENSURE(size == v.second, "Bad decimal field size: " << size);
- YQL_ENSURE(!NDecimal::IsError(v.first), "Bad decimal field data: " << data);
- return NUdf::TUnboxedValuePod(v.first);
- }
+ if (nativeYtTypeFlags & NTCF_DECIMAL) {
+ auto const params = static_cast<TDataDecimalType*>(type)->GetParams();
+ if (params.first < 10) {
+ i32 data;
+ buf.ReadMany((char*)&data, sizeof(data));
+ return NUdf::TUnboxedValuePod(NDecimal::FromYtDecimal(data));
+ } else if (params.first < 19) {
+ i64 data;
+ buf.ReadMany((char*)&data, sizeof(data));
+ return NUdf::TUnboxedValuePod(NDecimal::FromYtDecimal(data));
+ } else {
+ YQL_ENSURE(params.first < 36);
+ NDecimal::TInt128 data;
+ buf.ReadMany((char*)&data, sizeof(data));
+ return NUdf::TUnboxedValuePod(NDecimal::FromYtDecimal(data));
+ }
+ } else {
+ ui32 size;
+ buf.ReadMany(reinterpret_cast<char*>(&size), sizeof(size));
+ const auto maxSize = sizeof(NDecimal::TInt128);
+ YQL_ENSURE(size > 0U && size <= maxSize, "Bad decimal field size: " << size);
+ char data[maxSize];
+ buf.ReadMany(data, size);
+ const auto& v = NDecimal::Deserialize(data);
+ YQL_ENSURE(size == v.second, "Bad decimal field size: " << size);
+ YQL_ENSURE(!NDecimal::IsError(v.first), "Bad decimal field data: " << data);
+ return NUdf::TUnboxedValuePod(v.first);
+ }
}
case NUdf::TDataType<NUdf::TTzDate>::Id: {
@@ -1489,11 +1489,11 @@ NUdf::TUnboxedValue ReadSkiffData(TType* type, ui64 nativeYtTypeFlags, TInputBuf
}
}
-NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory, TInputBuf& buf)
-{
+NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory, TInputBuf& buf)
+{
if (type->IsData()) {
- return ReadSkiffData(type, nativeYtTypeFlags, buf);
+ return ReadSkiffData(type, nativeYtTypeFlags, buf);
}
if (type->IsOptional()) {
@@ -1502,7 +1502,7 @@ NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* ty
return NUdf::TUnboxedValue();
}
- auto value = ReadSkiffNativeYtValue(AS_TYPE(TOptionalType, type)->GetItemType(), nativeYtTypeFlags, holderFactory, buf);
+ auto value = ReadSkiffNativeYtValue(AS_TYPE(TOptionalType, type)->GetItemType(), nativeYtTypeFlags, holderFactory, buf);
return value.Release().MakeOptional();
}
@@ -1511,7 +1511,7 @@ NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* ty
NUdf::TUnboxedValue* items;
auto value = holderFactory.CreateDirectArrayHolder(tupleType->GetElementsCount(), items);
for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) {
- items[i] = ReadSkiffNativeYtValue(tupleType->GetElementType(i), nativeYtTypeFlags, holderFactory, buf);
+ items[i] = ReadSkiffNativeYtValue(tupleType->GetElementType(i), nativeYtTypeFlags, holderFactory, buf);
}
return value;
@@ -1521,17 +1521,17 @@ NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* ty
auto structType = AS_TYPE(TStructType, type);
NUdf::TUnboxedValue* items;
auto value = holderFactory.CreateDirectArrayHolder(structType->GetMembersCount(), items);
-
- if (auto cookie = type->GetCookie()) {
- const std::vector<size_t>& reorder = *((const std::vector<size_t>*)cookie);
- for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
- const auto ndx = reorder[i];
- items[ndx] = ReadSkiffNativeYtValue(structType->GetMemberType(ndx), nativeYtTypeFlags, holderFactory, buf);
- }
- } else {
- for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
- items[i] = ReadSkiffNativeYtValue(structType->GetMemberType(i), nativeYtTypeFlags, holderFactory, buf);
- }
+
+ if (auto cookie = type->GetCookie()) {
+ const std::vector<size_t>& reorder = *((const std::vector<size_t>*)cookie);
+ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
+ const auto ndx = reorder[i];
+ items[ndx] = ReadSkiffNativeYtValue(structType->GetMemberType(ndx), nativeYtTypeFlags, holderFactory, buf);
+ }
+ } else {
+ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
+ items[i] = ReadSkiffNativeYtValue(structType->GetMemberType(i), nativeYtTypeFlags, holderFactory, buf);
+ }
}
return value;
@@ -1541,7 +1541,7 @@ NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* ty
auto itemType = AS_TYPE(TListType, type)->GetItemType();
TDefaultListRepresentation items;
while (buf.Read() == '\0') {
- items = items.Append(ReadSkiffNativeYtValue(itemType, nativeYtTypeFlags, holderFactory, buf));
+ items = items.Append(ReadSkiffNativeYtValue(itemType, nativeYtTypeFlags, holderFactory, buf));
}
return holderFactory.CreateDirectListHolder(std::move(items));
@@ -1558,18 +1558,18 @@ NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* ty
if (varType->GetUnderlyingType()->IsTuple()) {
auto tupleType = AS_TYPE(TTupleType, varType->GetUnderlyingType());
YQL_ENSURE(data < tupleType->GetElementsCount());
- auto item = ReadSkiffNativeYtValue(tupleType->GetElementType(data), nativeYtTypeFlags, holderFactory, buf);
+ auto item = ReadSkiffNativeYtValue(tupleType->GetElementType(data), nativeYtTypeFlags, holderFactory, buf);
return holderFactory.CreateVariantHolder(item.Release(), data);
}
else {
auto structType = AS_TYPE(TStructType, varType->GetUnderlyingType());
- if (auto cookie = structType->GetCookie()) {
- const std::vector<size_t>& reorder = *((const std::vector<size_t>*)cookie);
- data = reorder[data];
- }
+ if (auto cookie = structType->GetCookie()) {
+ const std::vector<size_t>& reorder = *((const std::vector<size_t>*)cookie);
+ data = reorder[data];
+ }
YQL_ENSURE(data < structType->GetMembersCount());
-
- auto item = ReadSkiffNativeYtValue(structType->GetMemberType(data), nativeYtTypeFlags, holderFactory, buf);
+
+ auto item = ReadSkiffNativeYtValue(structType->GetMemberType(data), nativeYtTypeFlags, holderFactory, buf);
return holderFactory.CreateVariantHolder(item.Release(), data);
}
}
@@ -1593,8 +1593,8 @@ NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* ty
auto builder = holderFactory.NewDict(dictType, NUdf::TDictFlags::EDictKind::Hashed);
while (buf.Read() == '\0') {
- auto key = ReadSkiffNativeYtValue(keyType, nativeYtTypeFlags, holderFactory, buf);
- auto payload = ReadSkiffNativeYtValue(payloadType, nativeYtTypeFlags, holderFactory, buf);
+ auto key = ReadSkiffNativeYtValue(keyType, nativeYtTypeFlags, holderFactory, buf);
+ auto payload = ReadSkiffNativeYtValue(payloadType, nativeYtTypeFlags, holderFactory, buf);
builder->Add(std::move(key), std::move(payload));
}
@@ -1604,9 +1604,9 @@ NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* ty
YQL_ENSURE(false, "Unsupported type: " << type->GetKindAsStr());
}
-extern "C" void ReadContainerNativeYtValue(TType* type, ui64 nativeYtTypeFlags, const NKikimr::NMiniKQL::THolderFactory& holderFactory,
+extern "C" void ReadContainerNativeYtValue(TType* type, ui64 nativeYtTypeFlags, const NKikimr::NMiniKQL::THolderFactory& holderFactory,
NUdf::TUnboxedValue& value, TInputBuf& buf, bool wrapOptional) {
- auto tmp = ReadSkiffNativeYtValue(type, nativeYtTypeFlags, holderFactory, buf);
+ auto tmp = ReadSkiffNativeYtValue(type, nativeYtTypeFlags, holderFactory, buf);
if (!wrapOptional) {
value = std::move(tmp);
} else {
@@ -1641,10 +1641,10 @@ public:
}
}
- void SetRecordBoundaryCallback(std::function<void()> callback) override {
- Y_UNUSED(callback);
- }
-
+ void SetRecordBoundaryCallback(std::function<void()> callback) override {
+ Y_UNUSED(callback);
+ }
+
void WriteBlocks(TOutputBuf& buf) const {
auto current = Dummy_.Next_; // skip dummy node
while (current != &Dummy_) {
@@ -1667,8 +1667,8 @@ public:
return std::make_pair((char*)(header + 1), (char*)newPage + TAlignedPagePool::POOL_PAGE_SIZE);
}
- void ReturnBlock(size_t avail, std::optional<size_t> lastRecordBoundary) override {
- Y_UNUSED(lastRecordBoundary);
+ void ReturnBlock(size_t avail, std::optional<size_t> lastRecordBoundary) override {
+ Y_UNUSED(lastRecordBoundary);
YQL_ENSURE(avail <= TAlignedPagePool::POOL_PAGE_SIZE - sizeof(TPageHeader));
Last_->Avail_ = avail;
}
@@ -1985,11 +1985,11 @@ extern "C" void WriteYsonContainerValue(TType* type, const NUdf::TUnboxedValuePo
blockWriter.WriteBlocks(buf);
}
-extern "C" void WriteContainerNativeYtValue(TType* type, ui64 nativeYtTypeFlags, const NUdf::TUnboxedValuePod& value, TOutputBuf& buf) {
- WriteSkiffNativeYtValue(type, nativeYtTypeFlags, value, buf);
+extern "C" void WriteContainerNativeYtValue(TType* type, ui64 nativeYtTypeFlags, const NUdf::TUnboxedValuePod& value, TOutputBuf& buf) {
+ WriteSkiffNativeYtValue(type, nativeYtTypeFlags, value, buf);
}
-void WriteSkiffData(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf) {
+void WriteSkiffData(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf) {
auto schemeType = static_cast<TDataType*>(type)->GetSchemeType();
switch (schemeType) {
case NUdf::TDataType<bool>::Id: {
@@ -2076,26 +2076,26 @@ void WriteSkiffData(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, cons
}
case NUdf::TDataType<NUdf::TDecimal>::Id: {
- if (nativeYtTypeFlags & NTCF_DECIMAL) {
- auto const params = static_cast<TDataDecimalType*>(type)->GetParams();
- const NDecimal::TInt128 data128 = value.GetInt128();
- if (params.first < 10) {
- auto data = NDecimal::ToYtDecimal<i32>(data128);
- buf.WriteMany((const char*)&data, sizeof(data));
- } else if (params.first < 19) {
- auto data = NDecimal::ToYtDecimal<i64>(data128);
- buf.WriteMany((const char*)&data, sizeof(data));
- } else {
- YQL_ENSURE(params.first < 36);
- auto data = NDecimal::ToYtDecimal<NDecimal::TInt128>(data128);
- buf.WriteMany((const char*)&data, sizeof(data));
- }
- } else {
- char data[sizeof(NDecimal::TInt128)];
- const ui32 size = NDecimal::Serialize(value.GetInt128(), data);
- buf.WriteMany(reinterpret_cast<const char*>(&size), sizeof(size));
- buf.WriteMany(data, size);
- }
+ if (nativeYtTypeFlags & NTCF_DECIMAL) {
+ auto const params = static_cast<TDataDecimalType*>(type)->GetParams();
+ const NDecimal::TInt128 data128 = value.GetInt128();
+ if (params.first < 10) {
+ auto data = NDecimal::ToYtDecimal<i32>(data128);
+ buf.WriteMany((const char*)&data, sizeof(data));
+ } else if (params.first < 19) {
+ auto data = NDecimal::ToYtDecimal<i64>(data128);
+ buf.WriteMany((const char*)&data, sizeof(data));
+ } else {
+ YQL_ENSURE(params.first < 36);
+ auto data = NDecimal::ToYtDecimal<NDecimal::TInt128>(data128);
+ buf.WriteMany((const char*)&data, sizeof(data));
+ }
+ } else {
+ char data[sizeof(NDecimal::TInt128)];
+ const ui32 size = NDecimal::Serialize(value.GetInt128(), data);
+ buf.WriteMany(reinterpret_cast<const char*>(&size), sizeof(size));
+ buf.WriteMany(data, size);
+ }
break;
}
@@ -2143,9 +2143,9 @@ void WriteSkiffData(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, cons
}
}
-void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf) {
+void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf) {
if (type->IsData()) {
- WriteSkiffData(type, nativeYtTypeFlags, value, buf);
+ WriteSkiffData(type, nativeYtTypeFlags, value, buf);
} else if (type->IsOptional()) {
if (!value) {
buf.Write('\0');
@@ -2153,7 +2153,7 @@ void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFl
}
buf.Write('\1');
- WriteSkiffNativeYtValue(AS_TYPE(TOptionalType, type)->GetItemType(), nativeYtTypeFlags, value.GetOptionalValue(), buf);
+ WriteSkiffNativeYtValue(AS_TYPE(TOptionalType, type)->GetItemType(), nativeYtTypeFlags, value.GetOptionalValue(), buf);
} else if (type->IsList()) {
auto itemType = AS_TYPE(TListType, type)->GetItemType();
auto elements = value.GetElements();
@@ -2161,13 +2161,13 @@ void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFl
ui32 size = value.GetListLength();
for (ui32 i = 0; i < size; ++i) {
buf.Write('\0');
- WriteSkiffNativeYtValue(itemType, nativeYtTypeFlags, elements[i], buf);
+ WriteSkiffNativeYtValue(itemType, nativeYtTypeFlags, elements[i], buf);
}
} else {
NUdf::TUnboxedValue item;
for (auto iter = value.GetListIterator(); iter.Next(item); ) {
buf.Write('\0');
- WriteSkiffNativeYtValue(itemType, nativeYtTypeFlags, item, buf);
+ WriteSkiffNativeYtValue(itemType, nativeYtTypeFlags, item, buf);
}
}
@@ -2177,38 +2177,38 @@ void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFl
auto elements = value.GetElements();
if (elements) {
for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) {
- WriteSkiffNativeYtValue(tupleType->GetElementType(i), nativeYtTypeFlags, elements[i], buf);
+ WriteSkiffNativeYtValue(tupleType->GetElementType(i), nativeYtTypeFlags, elements[i], buf);
}
} else {
for (ui32 i = 0; i < tupleType->GetElementsCount(); ++i) {
- WriteSkiffNativeYtValue(tupleType->GetElementType(i), nativeYtTypeFlags, value.GetElement(i), buf);
+ WriteSkiffNativeYtValue(tupleType->GetElementType(i), nativeYtTypeFlags, value.GetElement(i), buf);
}
}
} else if (type->IsStruct()) {
auto structType = AS_TYPE(TStructType, type);
auto elements = value.GetElements();
- if (auto cookie = type->GetCookie()) {
- const std::vector<size_t>& reorder = *((const std::vector<size_t>*)cookie);
- if (elements) {
- for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
- const auto ndx = reorder[i];
- WriteSkiffNativeYtValue(structType->GetMemberType(ndx), nativeYtTypeFlags, elements[ndx], buf);
- }
- } else {
- for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
- const auto ndx = reorder[i];
- WriteSkiffNativeYtValue(structType->GetMemberType(ndx), nativeYtTypeFlags, value.GetElement(ndx), buf);
- }
+ if (auto cookie = type->GetCookie()) {
+ const std::vector<size_t>& reorder = *((const std::vector<size_t>*)cookie);
+ if (elements) {
+ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
+ const auto ndx = reorder[i];
+ WriteSkiffNativeYtValue(structType->GetMemberType(ndx), nativeYtTypeFlags, elements[ndx], buf);
+ }
+ } else {
+ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
+ const auto ndx = reorder[i];
+ WriteSkiffNativeYtValue(structType->GetMemberType(ndx), nativeYtTypeFlags, value.GetElement(ndx), buf);
+ }
}
} else {
- if (elements) {
- for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
- WriteSkiffNativeYtValue(structType->GetMemberType(i), nativeYtTypeFlags, elements[i], buf);
- }
- } else {
- for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
- WriteSkiffNativeYtValue(structType->GetMemberType(i), nativeYtTypeFlags, value.GetElement(i), buf);
- }
+ if (elements) {
+ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
+ WriteSkiffNativeYtValue(structType->GetMemberType(i), nativeYtTypeFlags, elements[i], buf);
+ }
+ } else {
+ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
+ WriteSkiffNativeYtValue(structType->GetMemberType(i), nativeYtTypeFlags, value.GetElement(i), buf);
+ }
}
}
} else if (type->IsVariant()) {
@@ -2222,16 +2222,16 @@ void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFl
if (varType->GetUnderlyingType()->IsTuple()) {
auto tupleType = AS_TYPE(TTupleType, varType->GetUnderlyingType());
- WriteSkiffNativeYtValue(tupleType->GetElementType(index), nativeYtTypeFlags, value.GetVariantItem(), buf);
+ WriteSkiffNativeYtValue(tupleType->GetElementType(index), nativeYtTypeFlags, value.GetVariantItem(), buf);
} else {
auto structType = AS_TYPE(TStructType, varType->GetUnderlyingType());
- if (auto cookie = structType->GetCookie()) {
- const std::vector<size_t>& reorder = *((const std::vector<size_t>*)cookie);
- index = reorder[index];
- }
- YQL_ENSURE(index < structType->GetMembersCount());
-
- WriteSkiffNativeYtValue(structType->GetMemberType(index), nativeYtTypeFlags, value.GetVariantItem(), buf);
+ if (auto cookie = structType->GetCookie()) {
+ const std::vector<size_t>& reorder = *((const std::vector<size_t>*)cookie);
+ index = reorder[index];
+ }
+ YQL_ENSURE(index < structType->GetMembersCount());
+
+ WriteSkiffNativeYtValue(structType->GetMemberType(index), nativeYtTypeFlags, value.GetVariantItem(), buf);
}
} else if (type->IsVoid() || type->IsNull() || type->IsEmptyList() || type->IsEmptyDict()) {
} else if (type->IsDict()) {
@@ -2241,8 +2241,8 @@ void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFl
NUdf::TUnboxedValue key, payload;
for (auto iter = value.GetDictIterator(); iter.NextPair(key, payload); ) {
buf.Write('\0');
- WriteSkiffNativeYtValue(keyType, nativeYtTypeFlags, key, buf);
- WriteSkiffNativeYtValue(payloadType, nativeYtTypeFlags, payload, buf);
+ WriteSkiffNativeYtValue(keyType, nativeYtTypeFlags, key, buf);
+ WriteSkiffNativeYtValue(payloadType, nativeYtTypeFlags, payload, buf);
}
buf.Write('\xff');
diff --git a/ydb/library/yql/providers/common/codec/yql_codec.h b/ydb/library/yql/providers/common/codec/yql_codec.h
index 512e01224c..2a162857c2 100644
--- a/ydb/library/yql/providers/common/codec/yql_codec.h
+++ b/ydb/library/yql/providers/common/codec/yql_codec.h
@@ -1,21 +1,21 @@
#pragma once
-#include "yql_codec_buf.h"
-
+#include "yql_codec_buf.h"
+
#include <ydb/library/yql/minikql/mkql_node.h>
#include <ydb/library/yql/minikql/mkql_function_registry.h>
#include <ydb/library/yql/minikql/mkql_program_builder.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/ast/yql_expr.h>
-#include <util/generic/strbuf.h>
-#include <util/generic/maybe.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/maybe.h>
#include <util/stream/output.h>
-
+
#include <library/cpp/yson/public.h>
-#include <list>
-#include <vector>
+#include <list>
+#include <vector>
namespace NYT {
class TNode;
@@ -39,18 +39,18 @@ TString WriteYsonValue(
);
TMaybe<TVector<ui32>> CreateStructPositions(
- NKikimr::NMiniKQL::TType* inputType,
+ NKikimr::NMiniKQL::TType* inputType,
const TVector<TString>* columns = nullptr
-);
-
+);
+
NYT::TNode ValueToNode(const NKikimr::NUdf::TUnboxedValuePod& value, NKikimr::NMiniKQL::TType* type);
TExprNode::TPtr NodeToExprLiteral(TPositionHandle pos, const TTypeAnnotationNode& type, const NYT::TNode& node, TExprContext& ctx);
-
+
struct TCodecContext {
const NKikimr::NMiniKQL::TTypeEnvironment& Env;
NKikimr::NMiniKQL::TProgramBuilder Builder;
const NKikimr::NMiniKQL::THolderFactory* HolderFactory; // lazy initialized
- std::list<std::vector<size_t>> StructReorders;
+ std::list<std::vector<size_t>> StructReorders;
TCodecContext(
const NKikimr::NMiniKQL::TTypeEnvironment& env,
@@ -76,23 +76,23 @@ extern "C" void ReadYsonContainerValue(NKikimr::NMiniKQL::TType* type,
const NKikimr::NMiniKQL::THolderFactory& holderFactory, NKikimr::NUdf::TUnboxedValue& value, NCommon::TInputBuf& buf,
bool wrapOptional);
-NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
- const NKikimr::NMiniKQL::THolderFactory& holderFactory, TInputBuf& buf);
+NKikimr::NUdf::TUnboxedValue ReadSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
+ const NKikimr::NMiniKQL::THolderFactory& holderFactory, TInputBuf& buf);
-NKikimr::NUdf::TUnboxedValue ReadSkiffData(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, NCommon::TInputBuf& buf);
-extern "C" void ReadContainerNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
+NKikimr::NUdf::TUnboxedValue ReadSkiffData(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, NCommon::TInputBuf& buf);
+extern "C" void ReadContainerNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
const NKikimr::NMiniKQL::THolderFactory& holderFactory, NKikimr::NUdf::TUnboxedValue& value, NCommon::TInputBuf& buf,
bool wrapOptional);
extern "C" void WriteYsonContainerValue(NKikimr::NMiniKQL::TType* type,
const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf);
-void WriteSkiffData(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf);
+void WriteSkiffData(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags, const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf);
-void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
+void WriteSkiffNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf);
-extern "C" void WriteContainerNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
+extern "C" void WriteContainerNativeYtValue(NKikimr::NMiniKQL::TType* type, ui64 nativeYtTypeFlags,
const NKikimr::NUdf::TUnboxedValuePod& value, NCommon::TOutputBuf& buf);
void WriteYsonValueInTableFormat(TOutputBuf& buf, NKikimr::NMiniKQL::TType* type, const NKikimr::NUdf::TUnboxedValuePod& value, bool topLevel);
diff --git a/ydb/library/yql/providers/common/codec/yql_codec_buf.cpp b/ydb/library/yql/providers/common/codec/yql_codec_buf.cpp
index ddaaeab6b7..c99c1f89ae 100644
--- a/ydb/library/yql/providers/common/codec/yql_codec_buf.cpp
+++ b/ydb/library/yql/providers/common/codec/yql_codec_buf.cpp
@@ -83,7 +83,7 @@ TStringBuf TInputBuf::ReadYtString(ui32 lookAhead) {
void TInputBuf::Reset() {
if (End_) {
- Source_->ReturnBlock();
+ Source_->ReturnBlock();
}
Current_ = End_ = nullptr;
}
@@ -93,29 +93,29 @@ void TInputBuf::Fill() {
return;
}
- if (!Source_) {
- return;
- }
-
- bool blockSwitch = false;
+ if (!Source_) {
+ return;
+ }
+
+ bool blockSwitch = false;
if (End_) {
- Source_->ReturnBlock();
- blockSwitch = true;
+ Source_->ReturnBlock();
+ blockSwitch = true;
}
if (ReadTimer_) {
ReadTimer_->Acquire();
}
- std::tie(Current_, End_) = Source_->NextFilledBlock();
- blockSwitch = blockSwitch && Current_ != End_;
+ std::tie(Current_, End_) = Source_->NextFilledBlock();
+ blockSwitch = blockSwitch && Current_ != End_;
MKQL_ADD_STAT(JobStats_, InputBytes, End_ - Current_);
if (ReadTimer_) {
ReadTimer_->Release();
}
- if (blockSwitch && OnNextBlockCallback_) {
- OnNextBlockCallback_();
- }
+ if (blockSwitch && OnNextBlockCallback_) {
+ OnNextBlockCallback_();
+ }
}
bool TInputBuf::TryReadSlow(char& value) {
@@ -138,9 +138,9 @@ extern "C" void InputBufReadManySlowThunk(TInputBuf& in, char* buffer, size_t co
}
extern "C" void InputBufSkipManySlowThunk(TInputBuf& in, size_t count) {
- return in.SkipManySlow(count);
-}
-
+ return in.SkipManySlow(count);
+}
+
char TInputBuf::ReadSlow() {
End_ = Current_;
Fill();
@@ -225,18 +225,18 @@ void TOutputBuf::Flush() {
if (Current_ > Begin_) {
if (WriteTimer_) {
WriteTimer_->Acquire();
- }
- const ui64 avail = Current_ - Begin_;
- MKQL_ADD_STAT(JobStats_, OutputBytes, avail);
- WrittenBytes_ += avail;
- Target_.ReturnBlock(avail, RecordBoundary_ ? std::make_optional(RecordBoundary_ - Begin_) : std::nullopt);
- std::tie(Begin_, End_) = Target_.NextEmptyBlock();
- Current_ = Begin_;
- RecordBoundary_ = nullptr;
- if (WriteTimer_) {
+ }
+ const ui64 avail = Current_ - Begin_;
+ MKQL_ADD_STAT(JobStats_, OutputBytes, avail);
+ WrittenBytes_ += avail;
+ Target_.ReturnBlock(avail, RecordBoundary_ ? std::make_optional(RecordBoundary_ - Begin_) : std::nullopt);
+ std::tie(Begin_, End_) = Target_.NextEmptyBlock();
+ Current_ = Begin_;
+ RecordBoundary_ = nullptr;
+ if (WriteTimer_) {
WriteTimer_->Release();
}
- }
+ }
}
void TOutputBuf::WriteManySlow(const char* buffer, size_t count) {
diff --git a/ydb/library/yql/providers/common/codec/yql_codec_buf.h b/ydb/library/yql/providers/common/codec/yql_codec_buf.h
index d951a9010d..4dc6d63ff9 100644
--- a/ydb/library/yql/providers/common/codec/yql_codec_buf.h
+++ b/ydb/library/yql/providers/common/codec/yql_codec_buf.h
@@ -2,8 +2,8 @@
#include <ydb/library/yql/minikql/mkql_stats_registry.h>
#include <ydb/library/yql/utils/yql_panic.h>
-#include <library/cpp/yson/zigzag.h>
-
+#include <library/cpp/yson/zigzag.h>
+
#include <util/generic/yexception.h>
#include <util/generic/maybe.h>
#ifndef LLVM_BC
@@ -14,8 +14,8 @@ using TInstant = ui64;
#include <util/generic/vector.h>
#include <utility>
-#include <functional>
-#include <optional>
+#include <functional>
+#include <optional>
namespace NYql {
namespace NCommon {
@@ -35,10 +35,10 @@ struct IBlockReader {
struct IBlockWriter {
virtual ~IBlockWriter() = default;
- virtual void SetRecordBoundaryCallback(std::function<void()> callback) = 0;
-
+ virtual void SetRecordBoundaryCallback(std::function<void()> callback) = 0;
+
virtual std::pair<char*, char*> NextEmptyBlock() = 0;
- virtual void ReturnBlock(size_t avail, std::optional<size_t> lastRecordBoundary) = 0;
+ virtual void ReturnBlock(size_t avail, std::optional<size_t> lastRecordBoundary) = 0;
virtual void Finish() = 0;
};
@@ -56,31 +56,31 @@ friend void InputBufReadManySlowThunk(TInputBuf& in, char* buffer, size_t count)
friend void InputBufSkipManySlowThunk(TInputBuf& in, size_t count);
public:
- TInputBuf(NKikimr::NMiniKQL::TStatTimer* readTimer)
- : ReadTimer_(readTimer)
+ TInputBuf(NKikimr::NMiniKQL::TStatTimer* readTimer)
+ : ReadTimer_(readTimer)
{}
- TInputBuf(IBlockReader& source, NKikimr::NMiniKQL::TStatTimer* readTimer)
- : TInputBuf(readTimer)
- {
- SetSource(source);
- }
-
- void SetSource(IBlockReader& source) {
- Source_ = &source;
- Current_ = nullptr;
- End_ = nullptr;
- String_.clear();
- }
-
+ TInputBuf(IBlockReader& source, NKikimr::NMiniKQL::TStatTimer* readTimer)
+ : TInputBuf(readTimer)
+ {
+ SetSource(source);
+ }
+
+ void SetSource(IBlockReader& source) {
+ Source_ = &source;
+ Current_ = nullptr;
+ End_ = nullptr;
+ String_.clear();
+ }
+
void SetStats(NKikimr::NMiniKQL::IStatsRegistry* jobStats) {
JobStats_ = jobStats;
}
- void SetNextBlockCallback(std::function<void()> cb) {
- OnNextBlockCallback_ = std::move(cb);
- }
-
+ void SetNextBlockCallback(std::function<void()> cb) {
+ OnNextBlockCallback_ = std::move(cb);
+ }
+
bool TryRead(char& value) {
if (Y_LIKELY(Current_ < End_)) {
value = *Current_++;
@@ -180,13 +180,13 @@ private:
void SkipManySlow(size_t count);
private:
- IBlockReader* Source_ = nullptr;
+ IBlockReader* Source_ = nullptr;
NKikimr::NMiniKQL::TStatTimer* ReadTimer_;
NKikimr::NMiniKQL::IStatsRegistry* JobStats_ = nullptr;
const char* Current_ = nullptr;
const char* End_ = nullptr;
TVector<char> String_;
- std::function<void()> OnNextBlockCallback_;
+ std::function<void()> OnNextBlockCallback_;
};
//////////////////////////////////////////////////////////////////////////////////////////////////////////
@@ -259,10 +259,10 @@ public:
}
}
- void OnRecordBoundary() {
- RecordBoundary_ = Current_;
- }
-
+ void OnRecordBoundary() {
+ RecordBoundary_ = Current_;
+ }
+
private:
void WriteSlow(char value) {
OutputBufFlushThunk(*this);
@@ -278,7 +278,7 @@ private:
char* Begin_ = nullptr;
char* Current_ = nullptr;
char* End_ = nullptr;
- char* RecordBoundary_ = nullptr;
+ char* RecordBoundary_ = nullptr;
ui64 WrittenBytes_ = 0;
};
diff --git a/ydb/library/yql/providers/common/codec/yql_codec_results.cpp b/ydb/library/yql/providers/common/codec/yql_codec_results.cpp
index 96c7d784ca..6e52c04cab 100644
--- a/ydb/library/yql/providers/common/codec/yql_codec_results.cpp
+++ b/ydb/library/yql/providers/common/codec/yql_codec_results.cpp
@@ -13,7 +13,7 @@ void TYsonResultWriter::OnStringScalar(TStringBuf value) {
if (!IsUtf8(value)) {
TString encoded = Base64Encode(value);
Writer.OnBeginList();
- Writer.OnListItem();
+ Writer.OnListItem();
Writer.OnStringScalar(TStringBuf(encoded));
Writer.OnEndList();
} else {
diff --git a/ydb/library/yql/providers/common/codec/yql_codec_results.h b/ydb/library/yql/providers/common/codec/yql_codec_results.h
index e34d2a3a68..eab02e3d35 100644
--- a/ydb/library/yql/providers/common/codec/yql_codec_results.h
+++ b/ydb/library/yql/providers/common/codec/yql_codec_results.h
@@ -17,12 +17,12 @@ public:
public:
explicit TYsonResultWriter(NYson::TYsonConsumerBase& writer)
- : Writer(writer)
- {
- }
- void OnVoid() {
- Writer.OnStringScalar(VoidString);
- }
+ : Writer(writer)
+ {
+ }
+ void OnVoid() {
+ Writer.OnStringScalar(VoidString);
+ }
void OnNull() {
Writer.OnEntity();
}
@@ -34,55 +34,55 @@ public:
Writer.OnBeginList();
Writer.OnEndList();
}
- void OnEntity() {
- Writer.OnEntity();
- }
+ void OnEntity() {
+ Writer.OnEntity();
+ }
// numbers
- void OnInt64Scalar(i64 value) {
- WriteNumberAsString(value);
- }
- void OnUint64Scalar(ui64 value) {
- WriteNumberAsString(value);
- }
- void OnFloatScalar(float value) {
- Writer.OnStringScalar(::FloatToString(value));
- }
- void OnDoubleScalar(double value) {
- Writer.OnStringScalar(::FloatToString(value));
- }
- void OnBooleanScalar(bool value) {
- Writer.OnBooleanScalar(value);
- }
+ void OnInt64Scalar(i64 value) {
+ WriteNumberAsString(value);
+ }
+ void OnUint64Scalar(ui64 value) {
+ WriteNumberAsString(value);
+ }
+ void OnFloatScalar(float value) {
+ Writer.OnStringScalar(::FloatToString(value));
+ }
+ void OnDoubleScalar(double value) {
+ Writer.OnStringScalar(::FloatToString(value));
+ }
+ void OnBooleanScalar(bool value) {
+ Writer.OnBooleanScalar(value);
+ }
// strings
void OnStringScalar(TStringBuf value);
- void OnUtf8StringScalar(TStringBuf value) {
- Writer.OnStringScalar(value);
- }
+ void OnUtf8StringScalar(TStringBuf value) {
+ Writer.OnStringScalar(value);
+ }
// list construction
- void OnBeginList() {
- Writer.OnBeginList();
- }
- void OnListItem() {
- Writer.OnListItem();
- }
- void OnEndList() {
- Writer.OnEndList();
- }
- void OnBeginMap() {
- Writer.OnBeginMap();
- }
- void OnKeyedItem(const TStringBuf& key) {
- Writer.OnKeyedItem(key);
- }
- void OnEndMap() {
- Writer.OnEndMap();
- }
- void OnBeginAttributes() {
- Writer.OnBeginAttributes();
- }
- void OnEndAttributes() {
- Writer.OnEndAttributes();
- }
+ void OnBeginList() {
+ Writer.OnBeginList();
+ }
+ void OnListItem() {
+ Writer.OnListItem();
+ }
+ void OnEndList() {
+ Writer.OnEndList();
+ }
+ void OnBeginMap() {
+ Writer.OnBeginMap();
+ }
+ void OnKeyedItem(const TStringBuf& key) {
+ Writer.OnKeyedItem(key);
+ }
+ void OnEndMap() {
+ Writer.OnEndMap();
+ }
+ void OnBeginAttributes() {
+ Writer.OnBeginAttributes();
+ }
+ void OnEndAttributes() {
+ Writer.OnEndAttributes();
+ }
private:
template <typename T>
diff --git a/ydb/library/yql/providers/common/codec/yql_codec_type_flags.cpp b/ydb/library/yql/providers/common/codec/yql_codec_type_flags.cpp
index 410e7dfbd7..abf78956ab 100644
--- a/ydb/library/yql/providers/common/codec/yql_codec_type_flags.cpp
+++ b/ydb/library/yql/providers/common/codec/yql_codec_type_flags.cpp
@@ -1 +1 @@
-#include "yql_codec_type_flags.h"
+#include "yql_codec_type_flags.h"
diff --git a/ydb/library/yql/providers/common/codec/yql_codec_type_flags.h b/ydb/library/yql/providers/common/codec/yql_codec_type_flags.h
index cd593e8175..e4614f7111 100644
--- a/ydb/library/yql/providers/common/codec/yql_codec_type_flags.h
+++ b/ydb/library/yql/providers/common/codec/yql_codec_type_flags.h
@@ -1,28 +1,28 @@
-#pragma once
-
-#include <util/system/types.h>
-
-namespace NYql {
-
-enum ENativeTypeCompatFlags: ui64 {
- NTCF_COMPLEX = 1ull << 0 /* "complex" */,
- NTCF_DATE = 1ull << 1 /* "date" */,
- NTCF_NULL = 1ull << 2 /* "null" */,
- NTCF_VOID = 1ull << 3 /* "void" */,
- NTCF_FLOAT = 1ull << 4 /* "float" */,
- NTCF_JSON = 1ull << 5 /* "json" */,
- NTCF_DECIMAL = 1ull << 6 /* "decimal" */,
-
- NTCF_NO_YT_SUPPORT = 1ull << 40 /* "_no_yt_support" */,
-
- NTCF_NONE = 0ull /* "none" */,
- NTCF_LEGACY = NTCF_COMPLEX | NTCF_DATE | NTCF_NULL | NTCF_VOID /* "legacy" */,
-
- // All supported types by all YT production clusters
- NTCF_PRODUCTION = NTCF_COMPLEX | NTCF_DATE | NTCF_NULL | NTCF_VOID | NTCF_FLOAT | NTCF_JSON | NTCF_DECIMAL /* "production" */,
-
- // add all new types here, supported by YT
- NTCF_ALL = NTCF_PRODUCTION /* "all" */,
-};
-
-} // namespace NYql
+#pragma once
+
+#include <util/system/types.h>
+
+namespace NYql {
+
+enum ENativeTypeCompatFlags: ui64 {
+ NTCF_COMPLEX = 1ull << 0 /* "complex" */,
+ NTCF_DATE = 1ull << 1 /* "date" */,
+ NTCF_NULL = 1ull << 2 /* "null" */,
+ NTCF_VOID = 1ull << 3 /* "void" */,
+ NTCF_FLOAT = 1ull << 4 /* "float" */,
+ NTCF_JSON = 1ull << 5 /* "json" */,
+ NTCF_DECIMAL = 1ull << 6 /* "decimal" */,
+
+ NTCF_NO_YT_SUPPORT = 1ull << 40 /* "_no_yt_support" */,
+
+ NTCF_NONE = 0ull /* "none" */,
+ NTCF_LEGACY = NTCF_COMPLEX | NTCF_DATE | NTCF_NULL | NTCF_VOID /* "legacy" */,
+
+ // All supported types by all YT production clusters
+ NTCF_PRODUCTION = NTCF_COMPLEX | NTCF_DATE | NTCF_NULL | NTCF_VOID | NTCF_FLOAT | NTCF_JSON | NTCF_DECIMAL /* "production" */,
+
+ // add all new types here, supported by YT
+ NTCF_ALL = NTCF_PRODUCTION /* "all" */,
+};
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/common/comp_nodes/ya.make b/ydb/library/yql/providers/common/comp_nodes/ya.make
index 0498885783..4150bba69f 100644
--- a/ydb/library/yql/providers/common/comp_nodes/ya.make
+++ b/ydb/library/yql/providers/common/comp_nodes/ya.make
@@ -6,7 +6,7 @@ OWNER(
)
SRCS(
- yql_factory.h
+ yql_factory.h
yql_factory.cpp
yql_formatcode.h
yql_formatcode.cpp
diff --git a/ydb/library/yql/providers/common/comp_nodes/yql_factory.cpp b/ydb/library/yql/providers/common/comp_nodes/yql_factory.cpp
index 2f73058170..20c20d1b71 100644
--- a/ydb/library/yql/providers/common/comp_nodes/yql_factory.cpp
+++ b/ydb/library/yql/providers/common/comp_nodes/yql_factory.cpp
@@ -1,4 +1,4 @@
-#include "yql_factory.h"
+#include "yql_factory.h"
#include "yql_formatcode.h"
#include "yql_formattype.h"
#include "yql_makecode.h"
@@ -75,16 +75,16 @@ TComputationNodeFactory GetYqlFactory(ui32 exprCtxMutableIndex) {
};
}
-TComputationNodeFactory GetYqlFactory() {
- TComputationNodeFactory yqlFactory;
- return [yqlFactory]
- (TCallable& callable, const TComputationNodeFactoryContext& ctx) mutable -> IComputationNode* {
- if (!yqlFactory) {
- yqlFactory = GetYqlFactory(ctx.Mutables.CurValueIndex++);
- }
- return yqlFactory(callable, ctx);
- };
-}
-
+TComputationNodeFactory GetYqlFactory() {
+ TComputationNodeFactory yqlFactory;
+ return [yqlFactory]
+ (TCallable& callable, const TComputationNodeFactoryContext& ctx) mutable -> IComputationNode* {
+ if (!yqlFactory) {
+ yqlFactory = GetYqlFactory(ctx.Mutables.CurValueIndex++);
+ }
+ return yqlFactory(callable, ctx);
+ };
}
+
}
+}
diff --git a/ydb/library/yql/providers/common/comp_nodes/yql_factory.h b/ydb/library/yql/providers/common/comp_nodes/yql_factory.h
index 3adf1f2178..97c905bf08 100644
--- a/ydb/library/yql/providers/common/comp_nodes/yql_factory.h
+++ b/ydb/library/yql/providers/common/comp_nodes/yql_factory.h
@@ -5,7 +5,7 @@ namespace NKikimr {
namespace NMiniKQL {
TComputationNodeFactory GetYqlFactory(ui32 exprCtxMutableIndex);
-TComputationNodeFactory GetYqlFactory();
+TComputationNodeFactory GetYqlFactory();
}
}
diff --git a/ydb/library/yql/providers/common/comp_nodes/yql_maketype.cpp b/ydb/library/yql/providers/common/comp_nodes/yql_maketype.cpp
index fe3d3c626e..63cee3ed84 100644
--- a/ydb/library/yql/providers/common/comp_nodes/yql_maketype.cpp
+++ b/ydb/library/yql/providers/common/comp_nodes/yql_maketype.cpp
@@ -225,9 +225,9 @@ public:
auto tagType = exprCtxPtr->template MakeType<NYql::TTaggedExprType>(baseType, tagValue.AsStringRef());
if (!tagType->Validate(Pos_, *exprCtxPtr)) {
UdfTerminate(exprCtxPtr->IssueManager.GetIssues().ToString().data());
- }
-
- retType = tagType;
+ }
+
+ retType = tagType;
break;
}
diff --git a/ydb/library/yql/providers/common/comp_nodes/yql_reprcode.cpp b/ydb/library/yql/providers/common/comp_nodes/yql_reprcode.cpp
index a89bdd976d..7967ed8510 100644
--- a/ydb/library/yql/providers/common/comp_nodes/yql_reprcode.cpp
+++ b/ydb/library/yql/providers/common/comp_nodes/yql_reprcode.cpp
@@ -1,7 +1,7 @@
#include "yql_formatcode.h"
#include "yql_type_resource.h"
#include "yql_position.h"
-
+
#include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
@@ -23,7 +23,7 @@ public:
NUdf::TUnboxedValue DoCalculate(TComputationContext& ctx) const {
auto exprCtxPtr = GetExprContextPtr(ctx, ExprCtxMutableIndex_);
- const NYql::TTypeAnnotationNode* type = NYql::NCommon::ParseTypeFromYson(TStringBuf{Yson_}, *exprCtxPtr, Pos_);
+ const NYql::TTypeAnnotationNode* type = NYql::NCommon::ParseTypeFromYson(TStringBuf{Yson_}, *exprCtxPtr, Pos_);
auto value = Value_->GetValue(ctx);
auto node = NYql::NCommon::ValueToExprLiteral(type, value, *exprCtxPtr, exprCtxPtr->AppendPosition(Pos_));
return NUdf::TUnboxedValuePod(new TYqlCodeResource(exprCtxPtr, node));
diff --git a/ydb/library/yql/providers/common/comp_nodes/yql_serializetypehandle.cpp b/ydb/library/yql/providers/common/comp_nodes/yql_serializetypehandle.cpp
index 446b880812..87d7430920 100644
--- a/ydb/library/yql/providers/common/comp_nodes/yql_serializetypehandle.cpp
+++ b/ydb/library/yql/providers/common/comp_nodes/yql_serializetypehandle.cpp
@@ -1,6 +1,6 @@
#include "yql_serializetypehandle.h"
#include "yql_type_resource.h"
-
+
#include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node_impl.h>
#include <ydb/library/yql/minikql/mkql_string_util.h>
diff --git a/ydb/library/yql/providers/common/comp_nodes/yql_typehandle.cpp b/ydb/library/yql/providers/common/comp_nodes/yql_typehandle.cpp
index 229c7ffd1c..e2b963a53d 100644
--- a/ydb/library/yql/providers/common/comp_nodes/yql_typehandle.cpp
+++ b/ydb/library/yql/providers/common/comp_nodes/yql_typehandle.cpp
@@ -1,6 +1,6 @@
#include "yql_typehandle.h"
#include "yql_type_resource.h"
-
+
#include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h>
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
@@ -21,7 +21,7 @@ public:
NUdf::TUnboxedValue DoCalculate(TComputationContext& ctx) const {
auto exprCtxPtr = GetExprContextPtr(ctx, ExprCtxMutableIndex_);
- const NYql::TTypeAnnotationNode* type = NYql::NCommon::ParseTypeFromYson(TStringBuf{Yson_}, *exprCtxPtr);
+ const NYql::TTypeAnnotationNode* type = NYql::NCommon::ParseTypeFromYson(TStringBuf{Yson_}, *exprCtxPtr);
if (!type) {
UdfTerminate(exprCtxPtr->IssueManager.GetIssues().ToString().data());
}
diff --git a/ydb/library/yql/providers/common/config/ya.make b/ydb/library/yql/providers/common/config/ya.make
index 3211a5823b..0e07f07786 100644
--- a/ydb/library/yql/providers/common/config/ya.make
+++ b/ydb/library/yql/providers/common/config/ya.make
@@ -1,24 +1,24 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:yql)
-
-SRCS(
- yql_configuration_transformer.cpp
- yql_dispatch.cpp
- yql_setting.h
-)
-
-PEERDIR(
+
+SRCS(
+ yql_configuration_transformer.cpp
+ yql_dispatch.cpp
+ yql_setting.h
+)
+
+PEERDIR(
ydb/library/yql/core
ydb/library/yql/core/expr_nodes
ydb/library/yql/ast
- library/cpp/containers/sorted_vector
+ library/cpp/containers/sorted_vector
library/cpp/string_utils/parse_size
library/cpp/string_utils/levenshtein_diff
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/library/yql/providers/common/config/yql_configuration_transformer.cpp b/ydb/library/yql/providers/common/config/yql_configuration_transformer.cpp
index 9fbd44acb3..e53b6bd8b3 100644
--- a/ydb/library/yql/providers/common/config/yql_configuration_transformer.cpp
+++ b/ydb/library/yql/providers/common/config/yql_configuration_transformer.cpp
@@ -1,28 +1,28 @@
-#include "yql_configuration_transformer.h"
-
+#include "yql_configuration_transformer.h"
+
#include <ydb/library/yql/core/yql_graph_transformer.h>
#include <ydb/library/yql/core/yql_expr_optimize.h>
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
-
-#include <util/generic/maybe.h>
-
-namespace NYql {
-namespace NCommon {
-
-using namespace NNodes;
-
+
+#include <util/generic/maybe.h>
+
+namespace NYql {
+namespace NCommon {
+
+using namespace NNodes;
+
TProviderConfigurationTransformer::TProviderConfigurationTransformer(TSettingDispatcher::TPtr dispatcher,
- const TTypeAnnotationContext& types, const TString& provider, const THashSet<TStringBuf>& configureCallables)
+ const TTypeAnnotationContext& types, const TString& provider, const THashSet<TStringBuf>& configureCallables)
: Dispatcher(dispatcher)
, Types(types)
- , Provider(provider)
- , ConfigureCallables(configureCallables)
-{
- if (ConfigureCallables.empty()) {
- ConfigureCallables.insert(TCoConfigure::CallableName());
- }
-}
+ , Provider(provider)
+ , ConfigureCallables(configureCallables)
+{
+ if (ConfigureCallables.empty()) {
+ ConfigureCallables.insert(TCoConfigure::CallableName());
+ }
+}
IGraphTransformer::TStatus TProviderConfigurationTransformer::DoTransform(TExprNode::TPtr input,
TExprNode::TPtr& output, TExprContext& ctx)
@@ -30,27 +30,27 @@ IGraphTransformer::TStatus TProviderConfigurationTransformer::DoTransform(TExprN
output = input;
if (ctx.Step.IsDone(TExprStep::Configure)) {
return TStatus::Ok;
- }
-
+ }
+
TOptimizeExprSettings settings(nullptr);
settings.VisitChanges = true;
auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
- if (node->IsCallable(ConfigureCallables)) {
+ if (node->IsCallable(ConfigureCallables)) {
if (!EnsureMinArgsCount(*node, 2, ctx)) {
return nullptr;
}
-
- if (!TCoDataSource::Match(node->Child(TCoConfigure::idx_DataSource))) {
+
+ if (!TCoDataSource::Match(node->Child(TCoConfigure::idx_DataSource))) {
return node;
}
- auto ds = node->Child(TCoConfigure::idx_DataSource);
- if (ds->Child(TCoDataSource::idx_Category)->Content() != Provider) {
+ auto ds = node->Child(TCoConfigure::idx_DataSource);
+ if (ds->Child(TCoDataSource::idx_Category)->Content() != Provider) {
return node;
}
- if (!EnsureMinArgsCount(*ds, 2, ctx)) {
+ if (!EnsureMinArgsCount(*ds, 2, ctx)) {
return nullptr;
}
- if (!EnsureAtom(*ds->Child(1), ctx)) {
+ if (!EnsureAtom(*ds->Child(1), ctx)) {
return nullptr;
}
auto clusterName = TString(ds->Child(1)->Content());
@@ -58,70 +58,70 @@ IGraphTransformer::TStatus TProviderConfigurationTransformer::DoTransform(TExprN
if (!EnsureMinArgsCount(*node, 3, ctx)) {
return nullptr;
}
- if (!EnsureAtom(*node->Child(2), ctx)) {
+ if (!EnsureAtom(*node->Child(2), ctx)) {
return nullptr;
}
- auto atom = node->Child(2)->Content();
+ auto atom = node->Child(2)->Content();
if (atom == TStringBuf("Attr")) {
if (!EnsureMinArgsCount(*node, 4, ctx)) {
- return nullptr;
- }
-
+ return nullptr;
+ }
+
if (!EnsureMaxArgsCount(*node, 5, ctx)) {
- return nullptr;
- }
-
- if (!EnsureAtom(*node->Child(3), ctx)) {
- return nullptr;
- }
+ return nullptr;
+ }
+ if (!EnsureAtom(*node->Child(3), ctx)) {
+ return nullptr;
+ }
+
auto name = TString(node->Child(3)->Content());
if (name.StartsWith('_')) {
ctx.AddError(TIssue(ctx.GetPosition(node->Child(3)->Pos()),
TStringBuilder() << "Failed to override system setting: " << name));
- return nullptr;
- }
-
+ return nullptr;
+ }
+
TMaybe<TString> value;
- if (node->ChildrenSize() == 5) {
- if (node->Child(4)->IsCallable("EvaluateAtom")) {
+ if (node->ChildrenSize() == 5) {
+ if (node->Child(4)->IsCallable("EvaluateAtom")) {
return node;
- }
-
- if (!EnsureAtom(*node->Child(4), ctx)) {
- return nullptr;
- }
-
+ }
+
+ if (!EnsureAtom(*node->Child(4), ctx)) {
+ return nullptr;
+ }
+
value = TString(node->Child(4)->Content());
}
-
- if (!HandleAttr(node->Child(3)->Pos(), clusterName, name, value, ctx)) {
+
+ if (!HandleAttr(node->Child(3)->Pos(), clusterName, name, value, ctx)) {
return nullptr;
}
} else if (atom == TStringBuf("Auth")) {
if (!EnsureArgsCount(*node, 4, ctx)) {
return nullptr;
}
-
- if (!EnsureAtom(*node->Child(3), ctx)) {
+
+ if (!EnsureAtom(*node->Child(3), ctx)) {
return nullptr;
- }
-
+ }
+
auto credAlias = TString(node->Child(3)->Content());
- if (!HandleAuth(node->Child(3)->Pos(), clusterName, credAlias, ctx)) {
- return nullptr;
- }
+ if (!HandleAuth(node->Child(3)->Pos(), clusterName, credAlias, ctx)) {
+ return nullptr;
+ }
} else {
ctx.AddError(TIssue(ctx.GetPosition(node->Child(2)->Pos()), TStringBuilder()
<< "Unsupported configuration option: " << atom));
return nullptr;
- }
+ }
}
-
+
return node;
}, ctx, settings);
-
+
return status;
}
@@ -141,25 +141,25 @@ bool TProviderConfigurationTransformer::HandleAuth(TPositionHandle pos, const TS
if (!cred) {
ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder() << "Unknown credential: " << alias));
return false;
- }
-
+ }
+
if (cred->Category != Provider) {
ctx.AddError(TIssue(ctx.GetPosition(pos), TStringBuilder()
<< "Mismatch credential category, expected: "
<< Provider << ", but found: " << cred->Category));
return false;
}
-
+
Dispatcher->Dispatch(cluster, "Auth", cred->Content, TSettingDispatcher::EStage::STATIC);
return true;
}
-THolder<IGraphTransformer> CreateProviderConfigurationTransformer(
- TSettingDispatcher::TPtr dispatcher,
- const TTypeAnnotationContext& types,
- const TString& provider) {
+THolder<IGraphTransformer> CreateProviderConfigurationTransformer(
+ TSettingDispatcher::TPtr dispatcher,
+ const TTypeAnnotationContext& types,
+ const TString& provider) {
return THolder(new TProviderConfigurationTransformer(dispatcher, types, provider));
-}
-
-} // namespace NCommon
-} // namespace NYql
+}
+
+} // namespace NCommon
+} // namespace NYql
diff --git a/ydb/library/yql/providers/common/config/yql_configuration_transformer.h b/ydb/library/yql/providers/common/config/yql_configuration_transformer.h
index 38997103ff..ef9ee7aa30 100644
--- a/ydb/library/yql/providers/common/config/yql_configuration_transformer.h
+++ b/ydb/library/yql/providers/common/config/yql_configuration_transformer.h
@@ -1,21 +1,21 @@
-#pragma once
-
-#include "yql_dispatch.h"
-
+#pragma once
+
+#include "yql_dispatch.h"
+
#include <ydb/library/yql/core/yql_type_annotation.h>
-
-#include <util/generic/ptr.h>
-#include <util/generic/string.h>
-#include <util/generic/hash_set.h>
-#include <util/generic/strbuf.h>
-
-namespace NYql {
-namespace NCommon {
-
+
+#include <util/generic/ptr.h>
+#include <util/generic/string.h>
+#include <util/generic/hash_set.h>
+#include <util/generic/strbuf.h>
+
+namespace NYql {
+namespace NCommon {
+
class TProviderConfigurationTransformer : public TSyncTransformerBase {
public:
TProviderConfigurationTransformer(TSettingDispatcher::TPtr dispatcher,const TTypeAnnotationContext& types,
- const TString& provider, const THashSet<TStringBuf>& configureCallables = {});
+ const TString& provider, const THashSet<TStringBuf>& configureCallables = {});
TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final;
@@ -28,14 +28,14 @@ protected:
TSettingDispatcher::TPtr Dispatcher;
const TTypeAnnotationContext& Types;
TString Provider;
- THashSet<TStringBuf> ConfigureCallables;
+ THashSet<TStringBuf> ConfigureCallables;
};
-THolder<IGraphTransformer> CreateProviderConfigurationTransformer(
- TSettingDispatcher::TPtr dispatcher,
- const TTypeAnnotationContext& types,
- const TString& provider
-);
-
-} // namespace NCommon
-} // namespace NYql
+THolder<IGraphTransformer> CreateProviderConfigurationTransformer(
+ TSettingDispatcher::TPtr dispatcher,
+ const TTypeAnnotationContext& types,
+ const TString& provider
+);
+
+} // namespace NCommon
+} // namespace NYql
diff --git a/ydb/library/yql/providers/common/config/yql_dispatch.cpp b/ydb/library/yql/providers/common/config/yql_dispatch.cpp
index 4e6df45a8f..eb3b069d74 100644
--- a/ydb/library/yql/providers/common/config/yql_dispatch.cpp
+++ b/ydb/library/yql/providers/common/config/yql_dispatch.cpp
@@ -1,193 +1,193 @@
-#include "yql_dispatch.h"
-
+#include "yql_dispatch.h"
+
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
#include <ydb/library/yql/ast/yql_expr.h>
-
+
#include <library/cpp/string_utils/levenshtein_diff/levenshtein_diff.h>
-
-#include <util/string/split.h>
-#include <util/random/random.h>
-#include <util/datetime/base.h>
-
-namespace NYql {
-
-namespace NPrivate {
-
-template <>
-TParser<TString> GetDefaultParser<TString>() {
- return [] (const TString& str) {
- return str;
- };
-}
-
-template<>
-TParser<bool> GetDefaultParser<bool>() {
- // Special handle of empty value to properly set setting for empty pragmas like yt.UseFeautre;
- return [] (const TString& str) {
- return str ? FromString<bool>(str) : true;
- };
-}
-
-template <>
-TParser<TGUID> GetDefaultParser<TGUID>() {
- return [] (const TString& str) {
- TGUID guid;
- if (!GetGuid(str, guid)) {
- throw yexception() << "Bad GUID format";
- }
- return guid;
- };
-}
-
-template<>
-TParser<NSize::TSize> GetDefaultParser<NSize::TSize>() {
- return [] (const TString& str) -> NSize::TSize {
- return NSize::ParseSize(str); // Support suffixes k, m, g, ...
- };
-}
-
-template<>
-TParser<TInstant> GetDefaultParser<TInstant>() {
- return [] (const TString& str) {
- TInstant val;
+
+#include <util/string/split.h>
+#include <util/random/random.h>
+#include <util/datetime/base.h>
+
+namespace NYql {
+
+namespace NPrivate {
+
+template <>
+TParser<TString> GetDefaultParser<TString>() {
+ return [] (const TString& str) {
+ return str;
+ };
+}
+
+template<>
+TParser<bool> GetDefaultParser<bool>() {
+ // Special handle of empty value to properly set setting for empty pragmas like yt.UseFeautre;
+ return [] (const TString& str) {
+ return str ? FromString<bool>(str) : true;
+ };
+}
+
+template <>
+TParser<TGUID> GetDefaultParser<TGUID>() {
+ return [] (const TString& str) {
+ TGUID guid;
+ if (!GetGuid(str, guid)) {
+ throw yexception() << "Bad GUID format";
+ }
+ return guid;
+ };
+}
+
+template<>
+TParser<NSize::TSize> GetDefaultParser<NSize::TSize>() {
+ return [] (const TString& str) -> NSize::TSize {
+ return NSize::ParseSize(str); // Support suffixes k, m, g, ...
+ };
+}
+
+template<>
+TParser<TInstant> GetDefaultParser<TInstant>() {
+ return [] (const TString& str) {
+ TInstant val;
if (!TInstant::TryParseIso8601(str, val)
&& !TInstant::TryParseRfc822(str, val)
&& !TInstant::TryParseHttp(str, val)
&& !TInstant::TryParseX509(str, val)
- ) {
- throw yexception() << "Bad date/time format";
- }
- return val;
- };
-}
-
-#define YQL_DEFINE_PRIMITIVE_SETTING_PARSER(type) \
- template<> \
- TParser<type> GetDefaultParser<type>() { \
- return [] (const TString& s) { return FromString<type>(s); }; \
- }
-
-#define YQL_DEFINE_CONTAINER_SETTING_PARSER(type) \
- template <> \
- TParser<type> GetDefaultParser<type>() { \
- return [] (const TString& str) { \
- type res; \
+ ) {
+ throw yexception() << "Bad date/time format";
+ }
+ return val;
+ };
+}
+
+#define YQL_DEFINE_PRIMITIVE_SETTING_PARSER(type) \
+ template<> \
+ TParser<type> GetDefaultParser<type>() { \
+ return [] (const TString& s) { return FromString<type>(s); }; \
+ }
+
+#define YQL_DEFINE_CONTAINER_SETTING_PARSER(type) \
+ template <> \
+ TParser<type> GetDefaultParser<type>() { \
+ return [] (const TString& str) { \
+ type res; \
StringSplitter(str).SplitBySet(",;| ").AddTo(&res); \
- for (auto& s: res) { \
- if (s.empty()) { \
- throw yexception() << "Empty value item"; \
- } \
- } \
- return res; \
- }; \
- }
-
-YQL_PRIMITIVE_SETTING_PARSER_TYPES(YQL_DEFINE_PRIMITIVE_SETTING_PARSER)
-YQL_CONTAINER_SETTING_PARSER_TYPES(YQL_DEFINE_CONTAINER_SETTING_PARSER)
-
-} // NPrivate
-
-namespace NCommon {
-
-bool TSettingDispatcher::Dispatch(const TString& cluster, const TString& name, const TMaybe<TString>& value, EStage stage) {
- auto normalizedName = NormalizeName(name);
- if (auto handler = Handlers.Value(normalizedName, TSettingHandler::TPtr())) {
- if (cluster != ALL_CLUSTERS) {
- if (!handler->IsRuntime()) {
- ythrow yexception() << "Static setting " << name.Quote() << " cannot be set for specific cluster";
- }
+ for (auto& s: res) { \
+ if (s.empty()) { \
+ throw yexception() << "Empty value item"; \
+ } \
+ } \
+ return res; \
+ }; \
+ }
+
+YQL_PRIMITIVE_SETTING_PARSER_TYPES(YQL_DEFINE_PRIMITIVE_SETTING_PARSER)
+YQL_CONTAINER_SETTING_PARSER_TYPES(YQL_DEFINE_CONTAINER_SETTING_PARSER)
+
+} // NPrivate
+
+namespace NCommon {
+
+bool TSettingDispatcher::Dispatch(const TString& cluster, const TString& name, const TMaybe<TString>& value, EStage stage) {
+ auto normalizedName = NormalizeName(name);
+ if (auto handler = Handlers.Value(normalizedName, TSettingHandler::TPtr())) {
+ if (cluster != ALL_CLUSTERS) {
+ if (!handler->IsRuntime()) {
+ ythrow yexception() << "Static setting " << name.Quote() << " cannot be set for specific cluster";
+ }
if (!ValidClusters.contains(cluster)) {
- TStringBuilder nearClusterMsg;
- for (auto& item: ValidClusters) {
+ TStringBuilder nearClusterMsg;
+ for (auto& item: ValidClusters) {
if (NLevenshtein::Distance(cluster, item) < DefaultMistypeDistance) {
- nearClusterMsg << ", did you mean " << item.Quote() << '?';
- break;
- }
- }
- ythrow yexception() << "Unknown cluster name " << cluster.Quote()
- << " for setting " << name.Quote() << nearClusterMsg;
- }
- }
- if (!value && !handler->IsRuntime()) {
- ythrow yexception() << "Static setting " << name.Quote() << " cannot be reset to default";
- }
-
- bool validateOnly = true;
- switch (stage) {
- case EStage::RUNTIME:
- validateOnly = !handler->IsRuntime();
- break;
- case EStage::STATIC:
- validateOnly = handler->IsRuntime();
- break;
- case EStage::CONFIG:
- validateOnly = false;
- break;
- }
-
- handler->Handle(cluster, value, validateOnly);
- return !validateOnly;
- } else {
+ nearClusterMsg << ", did you mean " << item.Quote() << '?';
+ break;
+ }
+ }
+ ythrow yexception() << "Unknown cluster name " << cluster.Quote()
+ << " for setting " << name.Quote() << nearClusterMsg;
+ }
+ }
+ if (!value && !handler->IsRuntime()) {
+ ythrow yexception() << "Static setting " << name.Quote() << " cannot be reset to default";
+ }
+
+ bool validateOnly = true;
+ switch (stage) {
+ case EStage::RUNTIME:
+ validateOnly = !handler->IsRuntime();
+ break;
+ case EStage::STATIC:
+ validateOnly = handler->IsRuntime();
+ break;
+ case EStage::CONFIG:
+ validateOnly = false;
+ break;
+ }
+
+ handler->Handle(cluster, value, validateOnly);
+ return !validateOnly;
+ } else {
// ignore unknown names in config
if (stage == EStage::CONFIG) {
return false;
}
- TStringBuilder nearHandlerMsg;
- for (auto& item: Handlers) {
+ TStringBuilder nearHandlerMsg;
+ for (auto& item: Handlers) {
if (NLevenshtein::Distance(normalizedName, item.first) < DefaultMistypeDistance) {
- nearHandlerMsg << ", did you mean " << item.second->GetDisplayName().Quote() << '?';
- break;
- }
- }
- ythrow yexception() << "Unknown setting name " << name.Quote() << nearHandlerMsg;
- }
-}
-
-void TSettingDispatcher::FreezeDefaults() {
- for (auto& item: Handlers) {
- item.second->FreezeDefault();
- }
-}
-
+ nearHandlerMsg << ", did you mean " << item.second->GetDisplayName().Quote() << '?';
+ break;
+ }
+ }
+ ythrow yexception() << "Unknown setting name " << name.Quote() << nearHandlerMsg;
+ }
+}
+
+void TSettingDispatcher::FreezeDefaults() {
+ for (auto& item: Handlers) {
+ item.second->FreezeDefault();
+ }
+}
+
void TSettingDispatcher::Restore() {
for (auto& item: Handlers) {
item.second->Restore(ALL_CLUSTERS);
}
}
-template <class TActivation>
-bool TSettingDispatcher::Allow(const TActivation& activation, const TString& userName) {
- if (AnyOf(activation.GetIncludeUsers(), [&](const auto& user) { return user == userName; })) {
- return true;
- }
- if (AnyOf(activation.GetExcludeUsers(), [&](const auto& user) { return user == userName; })) {
- return false;
- }
- if (userName.StartsWith("robot") && activation.GetExcludeRobots()) {
- return false;
- }
-
- ui32 percent = activation.GetPercentage();
- if (activation.ByHourSize()) {
- auto now = TInstant::Now();
- struct tm local = {};
- now.LocalTime(&local);
- const auto hour = ui32(local.tm_hour);
-
- for (auto& byHour: activation.GetByHour()) {
- if (byHour.GetHour() == hour) {
- percent = byHour.GetPercentage();
- break;
- }
- }
- }
- const auto random = RandomNumber<ui8>(100);
- return random < percent;
-}
-
-template bool TSettingDispatcher::Allow<NYql::TActivationPercentage>(const NYql::TActivationPercentage& activation, const TString& userName);
-
-} // NCommon
-} // NYql
+template <class TActivation>
+bool TSettingDispatcher::Allow(const TActivation& activation, const TString& userName) {
+ if (AnyOf(activation.GetIncludeUsers(), [&](const auto& user) { return user == userName; })) {
+ return true;
+ }
+ if (AnyOf(activation.GetExcludeUsers(), [&](const auto& user) { return user == userName; })) {
+ return false;
+ }
+ if (userName.StartsWith("robot") && activation.GetExcludeRobots()) {
+ return false;
+ }
+
+ ui32 percent = activation.GetPercentage();
+ if (activation.ByHourSize()) {
+ auto now = TInstant::Now();
+ struct tm local = {};
+ now.LocalTime(&local);
+ const auto hour = ui32(local.tm_hour);
+
+ for (auto& byHour: activation.GetByHour()) {
+ if (byHour.GetHour() == hour) {
+ percent = byHour.GetPercentage();
+ break;
+ }
+ }
+ }
+ const auto random = RandomNumber<ui8>(100);
+ return random < percent;
+}
+
+template bool TSettingDispatcher::Allow<NYql::TActivationPercentage>(const NYql::TActivationPercentage& activation, const TString& userName);
+
+} // NCommon
+} // NYql
diff --git a/ydb/library/yql/providers/common/config/yql_dispatch.h b/ydb/library/yql/providers/common/config/yql_dispatch.h
index 7389cbf082..cf68553b53 100644
--- a/ydb/library/yql/providers/common/config/yql_dispatch.h
+++ b/ydb/library/yql/providers/common/config/yql_dispatch.h
@@ -1,85 +1,85 @@
-#pragma once
-
-#include "yql_setting.h"
-
+#pragma once
+
+#include "yql_setting.h"
+
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
-
+
#include <library/cpp/string_utils/parse_size/parse_size.h>
-
-#include <util/string/cast.h>
-#include <util/string/join.h>
-#include <util/datetime/base.h>
-#include <util/generic/ptr.h>
-#include <util/generic/string.h>
-#include <util/generic/strbuf.h>
-#include <util/generic/hash.h>
-#include <util/generic/hash_set.h>
-#include <util/generic/yexception.h>
-#include <util/generic/vector.h>
-#include <util/generic/guid.h>
-#include <util/generic/maybe.h>
-#include <util/generic/algorithm.h>
-
-#include <type_traits>
-
-namespace NYql {
-
-namespace NPrivate {
-
-template <typename TType>
-using TParser = std::function<TType(const TString&)>;
-
-template <typename TType>
-TParser<TType> GetDefaultParser() {
- return [] (const TString&) -> TType { throw yexception() << "Unsupported parser"; };
-}
-
-template <>
-TParser<TString> GetDefaultParser<TString>();
-
-template<>
-TParser<bool> GetDefaultParser<bool>();
-
-template <>
-TParser<TGUID> GetDefaultParser<TGUID>();
-
-template<>
-TParser<NSize::TSize> GetDefaultParser<NSize::TSize>();
-
-template<>
-TParser<TInstant> GetDefaultParser<TInstant>();
-
-
-#define YQL_PRIMITIVE_SETTING_PARSER_TYPES(XX) \
- XX(ui8) \
- XX(ui16) \
- XX(ui32) \
- XX(ui64) \
- XX(i8) \
- XX(i16) \
- XX(i32) \
- XX(i64) \
- XX(float) \
- XX(double) \
- XX(TDuration)
-
-#define YQL_CONTAINER_SETTING_PARSER_TYPES(XX) \
+
+#include <util/string/cast.h>
+#include <util/string/join.h>
+#include <util/datetime/base.h>
+#include <util/generic/ptr.h>
+#include <util/generic/string.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/hash.h>
+#include <util/generic/hash_set.h>
+#include <util/generic/yexception.h>
+#include <util/generic/vector.h>
+#include <util/generic/guid.h>
+#include <util/generic/maybe.h>
+#include <util/generic/algorithm.h>
+
+#include <type_traits>
+
+namespace NYql {
+
+namespace NPrivate {
+
+template <typename TType>
+using TParser = std::function<TType(const TString&)>;
+
+template <typename TType>
+TParser<TType> GetDefaultParser() {
+ return [] (const TString&) -> TType { throw yexception() << "Unsupported parser"; };
+}
+
+template <>
+TParser<TString> GetDefaultParser<TString>();
+
+template<>
+TParser<bool> GetDefaultParser<bool>();
+
+template <>
+TParser<TGUID> GetDefaultParser<TGUID>();
+
+template<>
+TParser<NSize::TSize> GetDefaultParser<NSize::TSize>();
+
+template<>
+TParser<TInstant> GetDefaultParser<TInstant>();
+
+
+#define YQL_PRIMITIVE_SETTING_PARSER_TYPES(XX) \
+ XX(ui8) \
+ XX(ui16) \
+ XX(ui32) \
+ XX(ui64) \
+ XX(i8) \
+ XX(i16) \
+ XX(i32) \
+ XX(i64) \
+ XX(float) \
+ XX(double) \
+ XX(TDuration)
+
+#define YQL_CONTAINER_SETTING_PARSER_TYPES(XX) \
XX(TVector<TString>) \
XX(TSet<TString>) \
XX(THashSet<TString>)
-
-#define YQL_DECLARE_SETTING_PARSER(type) \
- template<> \
- TParser<type> GetDefaultParser<type>();
-
-YQL_PRIMITIVE_SETTING_PARSER_TYPES(YQL_DECLARE_SETTING_PARSER)
-YQL_CONTAINER_SETTING_PARSER_TYPES(YQL_DECLARE_SETTING_PARSER)
-
+
+#define YQL_DECLARE_SETTING_PARSER(type) \
+ template<> \
+ TParser<type> GetDefaultParser<type>();
+
+YQL_PRIMITIVE_SETTING_PARSER_TYPES(YQL_DECLARE_SETTING_PARSER)
+YQL_CONTAINER_SETTING_PARSER_TYPES(YQL_DECLARE_SETTING_PARSER)
+
template<typename TType>
TMaybe<TType> GetValue(const NCommon::TConfSetting<TType, true>& setting, const TString& cluster) {
return setting.Get(cluster);
-}
-
+}
+
template<typename TType>
TMaybe<TType> GetValue(const NCommon::TConfSetting<TType, false>& setting, const TString& cluster) {
Y_UNUSED(cluster);
@@ -88,282 +88,282 @@ TMaybe<TType> GetValue(const NCommon::TConfSetting<TType, false>& setting, const
}
-namespace NCommon {
-
-class TSettingDispatcher: public TThrRefBase {
-public:
- using TPtr = TIntrusivePtr<TSettingDispatcher>;
-
- enum class EStage {
- CONFIG,
- STATIC,
- RUNTIME,
- };
-
- class TSettingHandler: public TThrRefBase {
- public:
- using TPtr = TIntrusivePtr<TSettingHandler>;
-
- TSettingHandler(const TString& name)
- : Name_(name)
- {
- }
-
- const TString& GetDisplayName() const {
- return Name_ ;
- }
-
+namespace NCommon {
+
+class TSettingDispatcher: public TThrRefBase {
+public:
+ using TPtr = TIntrusivePtr<TSettingDispatcher>;
+
+ enum class EStage {
+ CONFIG,
+ STATIC,
+ RUNTIME,
+ };
+
+ class TSettingHandler: public TThrRefBase {
+ public:
+ using TPtr = TIntrusivePtr<TSettingHandler>;
+
+ TSettingHandler(const TString& name)
+ : Name_(name)
+ {
+ }
+
+ const TString& GetDisplayName() const {
+ return Name_ ;
+ }
+
virtual void Handle(const TString& cluster, const TMaybe<TString>& value, bool validateOnly) = 0;
- virtual void FreezeDefault() = 0;
+ virtual void FreezeDefault() = 0;
virtual void Restore(const TString& cluster) = 0;
- virtual bool IsRuntime() const = 0;
-
- protected:
- TString Name_;
- };
-
- template <typename TType, bool RUNTIME>
- class TSettingHandlerImpl: public TSettingHandler {
- public:
- using TValueCallback = std::function<void(const TString&, TType)>;
-
- private:
- friend class TSettingDispatcher;
-
- TSettingHandlerImpl(const TString& name, TConfSetting<TType, RUNTIME>& setting)
- : TSettingHandler(name)
- , Setting_(setting)
- , Parser_(::NYql::NPrivate::GetDefaultParser<TType>())
- , ValueSetter_([this](const TString& cluster, TType value) {
- Setting_[cluster] = value;
- })
- {
- }
-
+ virtual bool IsRuntime() const = 0;
+
+ protected:
+ TString Name_;
+ };
+
+ template <typename TType, bool RUNTIME>
+ class TSettingHandlerImpl: public TSettingHandler {
+ public:
+ using TValueCallback = std::function<void(const TString&, TType)>;
+
+ private:
+ friend class TSettingDispatcher;
+
+ TSettingHandlerImpl(const TString& name, TConfSetting<TType, RUNTIME>& setting)
+ : TSettingHandler(name)
+ , Setting_(setting)
+ , Parser_(::NYql::NPrivate::GetDefaultParser<TType>())
+ , ValueSetter_([this](const TString& cluster, TType value) {
+ Setting_[cluster] = value;
+ })
+ {
+ }
+
void Handle(const TString& cluster, const TMaybe<TString>& value, bool validateOnly) override {
- try {
- if (value) {
- TType v = Parser_(*value);
-
- for (auto& validate: Validators_) {
- validate(cluster, v);
- }
- if (!validateOnly) {
- ValueSetter_(cluster, v);
- }
- } else if (!validateOnly) {
- Restore(cluster);
- }
- } catch (const yexception& e) {
- ythrow yexception() << "Bad " << Name_.Quote() << " setting for " << cluster.Quote() << " cluster: " << e.what();
- }
- }
-
- void FreezeDefault() override {
- Defaul_ = Setting_;
- }
-
+ try {
+ if (value) {
+ TType v = Parser_(*value);
+
+ for (auto& validate: Validators_) {
+ validate(cluster, v);
+ }
+ if (!validateOnly) {
+ ValueSetter_(cluster, v);
+ }
+ } else if (!validateOnly) {
+ Restore(cluster);
+ }
+ } catch (const yexception& e) {
+ ythrow yexception() << "Bad " << Name_.Quote() << " setting for " << cluster.Quote() << " cluster: " << e.what();
+ }
+ }
+
+ void FreezeDefault() override {
+ Defaul_ = Setting_;
+ }
+
void Restore(const TString& cluster) override {
- if (!Defaul_) {
- ythrow yexception() << "Cannot restore " << Name_.Quote() << " setting without freeze";
- }
- if (ALL_CLUSTERS == cluster) {
- Setting_ = Defaul_.GetRef();
- } else {
- if (auto value = NPrivate::GetValue(Defaul_.GetRef(), cluster)) {
- Setting_[cluster] = *value;
- } else {
- Setting_.Clear();
- }
- }
- }
-
- public:
- bool IsRuntime() const override {
- return Setting_.IsRuntime();
- }
-
- TSettingHandlerImpl& Lower(TType lower) {
- Validators_.push_back([lower](const TString&, TType value) {
- if (value < lower) {
- throw yexception() << "Value " << value << " is less than " << lower << " allowed lower bound";
- }
- });
- return *this;
- }
-
- TSettingHandlerImpl& Upper(TType upper) {
- Validators_.push_back([upper](const TString&, TType value) {
- if (value > upper) {
- throw yexception() << "Value " << value << " is greater than " << upper << " allowed upper bound";
- }
- });
- return *this;
- }
-
- template <class TContainer>
- TSettingHandlerImpl& Enum(const TContainer& container) {
+ if (!Defaul_) {
+ ythrow yexception() << "Cannot restore " << Name_.Quote() << " setting without freeze";
+ }
+ if (ALL_CLUSTERS == cluster) {
+ Setting_ = Defaul_.GetRef();
+ } else {
+ if (auto value = NPrivate::GetValue(Defaul_.GetRef(), cluster)) {
+ Setting_[cluster] = *value;
+ } else {
+ Setting_.Clear();
+ }
+ }
+ }
+
+ public:
+ bool IsRuntime() const override {
+ return Setting_.IsRuntime();
+ }
+
+ TSettingHandlerImpl& Lower(TType lower) {
+ Validators_.push_back([lower](const TString&, TType value) {
+ if (value < lower) {
+ throw yexception() << "Value " << value << " is less than " << lower << " allowed lower bound";
+ }
+ });
+ return *this;
+ }
+
+ TSettingHandlerImpl& Upper(TType upper) {
+ Validators_.push_back([upper](const TString&, TType value) {
+ if (value > upper) {
+ throw yexception() << "Value " << value << " is greater than " << upper << " allowed upper bound";
+ }
+ });
+ return *this;
+ }
+
+ template <class TContainer>
+ TSettingHandlerImpl& Enum(const TContainer& container) {
THashSet<TType> allowed(container.cbegin(), container.cend());
- Validators_.push_back([allowed = std::move(allowed)](const TString&, TType value) {
- if (!allowed.has(value)) {
+ Validators_.push_back([allowed = std::move(allowed)](const TString&, TType value) {
+ if (!allowed.has(value)) {
throw yexception() << "Value " << value << " is not in set of allowed values: " << JoinSeq(TStringBuf(","), allowed);
- }
- });
- return *this;
- }
-
- TSettingHandlerImpl& Enum(std::initializer_list<TType> list) {
+ }
+ });
+ return *this;
+ }
+
+ TSettingHandlerImpl& Enum(std::initializer_list<TType> list) {
THashSet<TType> allowed(list);
- Validators_.push_back([allowed = std::move(allowed)](const TString&, TType value) {
+ Validators_.push_back([allowed = std::move(allowed)](const TString&, TType value) {
if (!allowed.contains(value)) {
throw yexception() << "Value " << value << " is not in set of allowed values: " << JoinSeq(TStringBuf(","), allowed);
- }
- });
- return *this;
- }
-
- TSettingHandlerImpl& NonEmpty() {
- Validators_.push_back([](const TString&, TType value) {
- if (value.empty()) {
- throw yexception() << "Value is empty";
- }
- });
- return *this;
- }
-
- TSettingHandlerImpl& GlobalOnly() {
+ }
+ });
+ return *this;
+ }
+
+ TSettingHandlerImpl& NonEmpty() {
+ Validators_.push_back([](const TString&, TType value) {
+ if (value.empty()) {
+ throw yexception() << "Value is empty";
+ }
+ });
+ return *this;
+ }
+
+ TSettingHandlerImpl& GlobalOnly() {
Validators_.push_back([] (const TString& cluster, TType) {
- if (cluster != NCommon::ALL_CLUSTERS) {
- throw yexception() << "Option cannot be used with specific cluster";
- }
- });
- return *this;
- }
-
- TSettingHandlerImpl& Validator(TValueCallback&& validator) {
- Validators_.push_back(std::move(validator));
- return *this;
- }
-
- TSettingHandlerImpl& Validator(const TValueCallback& validator) {
- Validators_.push_back(validator);
- return *this;
- }
-
- TSettingHandlerImpl& Parser(::NYql::NPrivate::TParser<TType>&& parser) {
- Parser_ = std::move(parser);
- return *this;
- }
-
- TSettingHandlerImpl& Parser(const ::NYql::NPrivate::TParser<TType>& parser) {
- Parser_ = parser;
- return *this;
- }
-
- TSettingHandlerImpl& ValueSetter(TValueCallback&& hook) {
- ValueSetter_ = std::move(hook);
- return *this;
- }
-
- TSettingHandlerImpl& ValueSetter(const TValueCallback& hook) {
- ValueSetter_ = hook;
- return *this;
- }
-
- TSettingHandlerImpl& ValueSetterWithRestore(TValueCallback&& hook) {
- ValueSetter_ = [this, hook = std::move(hook)] (const TString& cluster, TType value) {
- if (Defaul_) {
- Restore(cluster);
- }
- hook(cluster, value);
- };
- return *this;
- }
-
- TSettingHandlerImpl& ValueSetterWithRestore(const TValueCallback& hook) {
- ValueSetter_ = [this, hook] (const TString& cluster, TType value) {
- if (Defaul_) {
- Restore(cluster);
- }
- hook(cluster, value);
- };
- return *this;
- }
-
- private:
- TConfSetting<TType, RUNTIME>& Setting_;
- TMaybe<TConfSetting<TType, RUNTIME>> Defaul_;
- ::NYql::NPrivate::TParser<TType> Parser_;
- TValueCallback ValueSetter_;
+ if (cluster != NCommon::ALL_CLUSTERS) {
+ throw yexception() << "Option cannot be used with specific cluster";
+ }
+ });
+ return *this;
+ }
+
+ TSettingHandlerImpl& Validator(TValueCallback&& validator) {
+ Validators_.push_back(std::move(validator));
+ return *this;
+ }
+
+ TSettingHandlerImpl& Validator(const TValueCallback& validator) {
+ Validators_.push_back(validator);
+ return *this;
+ }
+
+ TSettingHandlerImpl& Parser(::NYql::NPrivate::TParser<TType>&& parser) {
+ Parser_ = std::move(parser);
+ return *this;
+ }
+
+ TSettingHandlerImpl& Parser(const ::NYql::NPrivate::TParser<TType>& parser) {
+ Parser_ = parser;
+ return *this;
+ }
+
+ TSettingHandlerImpl& ValueSetter(TValueCallback&& hook) {
+ ValueSetter_ = std::move(hook);
+ return *this;
+ }
+
+ TSettingHandlerImpl& ValueSetter(const TValueCallback& hook) {
+ ValueSetter_ = hook;
+ return *this;
+ }
+
+ TSettingHandlerImpl& ValueSetterWithRestore(TValueCallback&& hook) {
+ ValueSetter_ = [this, hook = std::move(hook)] (const TString& cluster, TType value) {
+ if (Defaul_) {
+ Restore(cluster);
+ }
+ hook(cluster, value);
+ };
+ return *this;
+ }
+
+ TSettingHandlerImpl& ValueSetterWithRestore(const TValueCallback& hook) {
+ ValueSetter_ = [this, hook] (const TString& cluster, TType value) {
+ if (Defaul_) {
+ Restore(cluster);
+ }
+ hook(cluster, value);
+ };
+ return *this;
+ }
+
+ private:
+ TConfSetting<TType, RUNTIME>& Setting_;
+ TMaybe<TConfSetting<TType, RUNTIME>> Defaul_;
+ ::NYql::NPrivate::TParser<TType> Parser_;
+ TValueCallback ValueSetter_;
TVector<TValueCallback> Validators_;
- };
-
- TSettingDispatcher() = default;
- TSettingDispatcher(const TSettingDispatcher&) = delete;
-
- template <class TContainer>
- TSettingDispatcher(const TContainer& validClusters)
- : ValidClusters(validClusters.begin(), validClusters.end())
- {
- }
-
- template <class TContainer>
- void SetValidClusters(const TContainer& validClusters) {
- ValidClusters.clear();
- ValidClusters.insert(validClusters.begin(), validClusters.end());
- }
-
- template <typename TType, bool RUNTIME>
- TSettingHandlerImpl<TType, RUNTIME>& AddSetting(const TString& name, TConfSetting<TType, RUNTIME>& setting) {
- TIntrusivePtr<TSettingHandlerImpl<TType, RUNTIME>> handler = new TSettingHandlerImpl<TType, RUNTIME>(name, setting);
- if (!Handlers.insert({NormalizeName(name), handler}).second) {
- ythrow yexception() << "Duplicate configuration setting name " << name.Quote();
- }
- return *handler;
- }
-
- bool Dispatch(const TString& cluster, const TString& name, const TMaybe<TString>& value, EStage stage);
-
- template <class TContainer>
- void Dispatch(const TString& cluster, const TContainer& clusterValues, const TString& userName) {
- for (auto& v: clusterValues) {
- if (!v.HasActivation() || Allow(v.GetActivation(), userName)) {
- Dispatch(cluster, v.GetName(), v.GetValue(), EStage::CONFIG);
- }
- }
- }
-
- template <class TContainer>
- void Dispatch(const TString& cluster, const TContainer& clusterValues) {
- for (auto& v: clusterValues) {
- Dispatch(cluster, v.GetName(), v.GetValue(), EStage::CONFIG);
- }
- }
-
- template <class TContainer>
- void Dispatch(const TContainer& globalValues, const TString& userName) {
- Dispatch(ALL_CLUSTERS, globalValues, userName);
- }
-
- template <class TContainer>
- void Dispatch(const TContainer& globalValues) {
- Dispatch(ALL_CLUSTERS, globalValues);
- }
-
- void FreezeDefaults();
+ };
+
+ TSettingDispatcher() = default;
+ TSettingDispatcher(const TSettingDispatcher&) = delete;
+
+ template <class TContainer>
+ TSettingDispatcher(const TContainer& validClusters)
+ : ValidClusters(validClusters.begin(), validClusters.end())
+ {
+ }
+
+ template <class TContainer>
+ void SetValidClusters(const TContainer& validClusters) {
+ ValidClusters.clear();
+ ValidClusters.insert(validClusters.begin(), validClusters.end());
+ }
+
+ template <typename TType, bool RUNTIME>
+ TSettingHandlerImpl<TType, RUNTIME>& AddSetting(const TString& name, TConfSetting<TType, RUNTIME>& setting) {
+ TIntrusivePtr<TSettingHandlerImpl<TType, RUNTIME>> handler = new TSettingHandlerImpl<TType, RUNTIME>(name, setting);
+ if (!Handlers.insert({NormalizeName(name), handler}).second) {
+ ythrow yexception() << "Duplicate configuration setting name " << name.Quote();
+ }
+ return *handler;
+ }
+
+ bool Dispatch(const TString& cluster, const TString& name, const TMaybe<TString>& value, EStage stage);
+
+ template <class TContainer>
+ void Dispatch(const TString& cluster, const TContainer& clusterValues, const TString& userName) {
+ for (auto& v: clusterValues) {
+ if (!v.HasActivation() || Allow(v.GetActivation(), userName)) {
+ Dispatch(cluster, v.GetName(), v.GetValue(), EStage::CONFIG);
+ }
+ }
+ }
+
+ template <class TContainer>
+ void Dispatch(const TString& cluster, const TContainer& clusterValues) {
+ for (auto& v: clusterValues) {
+ Dispatch(cluster, v.GetName(), v.GetValue(), EStage::CONFIG);
+ }
+ }
+
+ template <class TContainer>
+ void Dispatch(const TContainer& globalValues, const TString& userName) {
+ Dispatch(ALL_CLUSTERS, globalValues, userName);
+ }
+
+ template <class TContainer>
+ void Dispatch(const TContainer& globalValues) {
+ Dispatch(ALL_CLUSTERS, globalValues);
+ }
+
+ void FreezeDefaults();
void Restore();
-
-protected:
- template <class TActivation>
- static bool Allow(const TActivation& activation, const TString& userName);
-
-protected:
+
+protected:
+ template <class TActivation>
+ static bool Allow(const TActivation& activation, const TString& userName);
+
+protected:
THashSet<TString> ValidClusters;
THashMap<TString, TSettingHandler::TPtr> Handlers;
-};
-
-} // namespace NCommon
-} // namespace NYql
-
-#define REGISTER_SETTING(dispatcher, setting) \
- (dispatcher).AddSetting(#setting, setting)
+};
+
+} // namespace NCommon
+} // namespace NYql
+
+#define REGISTER_SETTING(dispatcher, setting) \
+ (dispatcher).AddSetting(#setting, setting)
diff --git a/ydb/library/yql/providers/common/config/yql_setting.h b/ydb/library/yql/providers/common/config/yql_setting.h
index 148b211bb7..4e90c23794 100644
--- a/ydb/library/yql/providers/common/config/yql_setting.h
+++ b/ydb/library/yql/providers/common/config/yql_setting.h
@@ -1,129 +1,129 @@
-#pragma once
-
-#include <library/cpp/containers/sorted_vector/sorted_vector.h>
-
-#include <util/generic/string.h>
-#include <util/generic/strbuf.h>
-#include <util/generic/maybe.h>
-#include <util/generic/yexception.h>
-
-namespace NYql {
-namespace NCommon {
-
-const TString ALL_CLUSTERS = "$all";
-
-template <typename TType, bool RUNTIME = true>
-class TConfSetting {
-public:
- TConfSetting() = default;
- TConfSetting(const TType& value) {
- PerClusterValue[ALL_CLUSTERS] = value;
- }
- TConfSetting(const TConfSetting&) = default;
- TConfSetting(TConfSetting&&) = default;
- ~TConfSetting() = default;
-
- bool IsRuntime() const {
- return RUNTIME;
- }
-
- TType& operator[](const TString& cluster) {
- if (ALL_CLUSTERS == cluster) {
- PerClusterValue.clear();
- }
- return PerClusterValue[cluster];
- }
- TConfSetting& operator =(const TType& value) {
- PerClusterValue.clear();
- PerClusterValue[ALL_CLUSTERS] = value;
- return *this;
- }
- TConfSetting& operator =(const TConfSetting&) = default;
- TConfSetting& operator =(TConfSetting&&) = default;
-
- template <typename TFunc>
- void UpdateAll(TFunc func) {
- PerClusterValue[ALL_CLUSTERS]; // insert record for all clusters if it is not present
- for (auto& it: PerClusterValue) {
- func(it.first, it.second);
- }
- }
-
- TMaybe<TType> Get(const TString& cluster) const {
- if (!PerClusterValue.empty()) {
- auto it = PerClusterValue.find(cluster);
- if (it != PerClusterValue.end()) {
- return MakeMaybe(it->second);
- }
- it = PerClusterValue.find(ALL_CLUSTERS);
- if (it != PerClusterValue.end()) {
- return MakeMaybe(it->second);
- }
- }
- return Nothing();
- }
-
- void Clear() {
- PerClusterValue.clear();
- }
-
- void Clear(const TString& cluster) {
- if (ALL_CLUSTERS == cluster) {
- PerClusterValue.clear();
- } else {
- PerClusterValue.erase(cluster);
- }
- }
-
-private:
- NSorted::TSimpleMap<TString, TType> PerClusterValue; // Uses special '$all' key for all clusters
-};
-
-template <typename TType>
-class TConfSetting<TType, false> {
-public:
- TConfSetting() = default;
- TConfSetting(const TType& value)
- : Value(value)
- {
- }
- TConfSetting(const TConfSetting&) = default;
- TConfSetting(TConfSetting&&) = default;
- ~TConfSetting() = default;
-
- bool IsRuntime() const {
- return false;
- }
-
- TType& operator[](const TString& cluster) {
- if (cluster != ALL_CLUSTERS) {
- ythrow yexception() << "Static setting cannot be set for specific cluster";
- }
- Value.ConstructInPlace();
- return Value.GetRef();
- }
- TConfSetting& operator =(const TType& value) {
- Value = value;
- return *this;
- }
- TConfSetting& operator =(const TConfSetting&) = default;
- TConfSetting& operator =(TConfSetting&&) = default;
-
+#pragma once
+
+#include <library/cpp/containers/sorted_vector/sorted_vector.h>
+
+#include <util/generic/string.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/maybe.h>
+#include <util/generic/yexception.h>
+
+namespace NYql {
+namespace NCommon {
+
+const TString ALL_CLUSTERS = "$all";
+
+template <typename TType, bool RUNTIME = true>
+class TConfSetting {
+public:
+ TConfSetting() = default;
+ TConfSetting(const TType& value) {
+ PerClusterValue[ALL_CLUSTERS] = value;
+ }
+ TConfSetting(const TConfSetting&) = default;
+ TConfSetting(TConfSetting&&) = default;
+ ~TConfSetting() = default;
+
+ bool IsRuntime() const {
+ return RUNTIME;
+ }
+
+ TType& operator[](const TString& cluster) {
+ if (ALL_CLUSTERS == cluster) {
+ PerClusterValue.clear();
+ }
+ return PerClusterValue[cluster];
+ }
+ TConfSetting& operator =(const TType& value) {
+ PerClusterValue.clear();
+ PerClusterValue[ALL_CLUSTERS] = value;
+ return *this;
+ }
+ TConfSetting& operator =(const TConfSetting&) = default;
+ TConfSetting& operator =(TConfSetting&&) = default;
+
+ template <typename TFunc>
+ void UpdateAll(TFunc func) {
+ PerClusterValue[ALL_CLUSTERS]; // insert record for all clusters if it is not present
+ for (auto& it: PerClusterValue) {
+ func(it.first, it.second);
+ }
+ }
+
+ TMaybe<TType> Get(const TString& cluster) const {
+ if (!PerClusterValue.empty()) {
+ auto it = PerClusterValue.find(cluster);
+ if (it != PerClusterValue.end()) {
+ return MakeMaybe(it->second);
+ }
+ it = PerClusterValue.find(ALL_CLUSTERS);
+ if (it != PerClusterValue.end()) {
+ return MakeMaybe(it->second);
+ }
+ }
+ return Nothing();
+ }
+
+ void Clear() {
+ PerClusterValue.clear();
+ }
+
+ void Clear(const TString& cluster) {
+ if (ALL_CLUSTERS == cluster) {
+ PerClusterValue.clear();
+ } else {
+ PerClusterValue.erase(cluster);
+ }
+ }
+
+private:
+ NSorted::TSimpleMap<TString, TType> PerClusterValue; // Uses special '$all' key for all clusters
+};
+
+template <typename TType>
+class TConfSetting<TType, false> {
+public:
+ TConfSetting() = default;
+ TConfSetting(const TType& value)
+ : Value(value)
+ {
+ }
+ TConfSetting(const TConfSetting&) = default;
+ TConfSetting(TConfSetting&&) = default;
+ ~TConfSetting() = default;
+
+ bool IsRuntime() const {
+ return false;
+ }
+
+ TType& operator[](const TString& cluster) {
+ if (cluster != ALL_CLUSTERS) {
+ ythrow yexception() << "Static setting cannot be set for specific cluster";
+ }
+ Value.ConstructInPlace();
+ return Value.GetRef();
+ }
+ TConfSetting& operator =(const TType& value) {
+ Value = value;
+ return *this;
+ }
+ TConfSetting& operator =(const TConfSetting&) = default;
+ TConfSetting& operator =(TConfSetting&&) = default;
+
TMaybe<TType> Get() const {
- return Value;
- }
-
- void Clear() {
- Value.Clear();
- }
-
- void Clear(const TString&) {
- Value.Clear();
- }
-
-private:
- TMaybe<TType> Value;
-};
-
-} // namespace NCommon
-} // namespace NYql
+ return Value;
+ }
+
+ void Clear() {
+ Value.Clear();
+ }
+
+ void Clear(const TString&) {
+ Value.Clear();
+ }
+
+private:
+ TMaybe<TType> Value;
+};
+
+} // namespace NCommon
+} // namespace NYql
diff --git a/ydb/library/yql/providers/common/dq/ya.make b/ydb/library/yql/providers/common/dq/ya.make
index 495e3bbaef..6de1ca2d3b 100644
--- a/ydb/library/yql/providers/common/dq/ya.make
+++ b/ydb/library/yql/providers/common/dq/ya.make
@@ -1,15 +1,15 @@
-LIBRARY()
-
-OWNER(
- g:yql
-)
-
-SRCS(
- yql_dq_integration_impl.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ g:yql
+)
+
+SRCS(
+ yql_dq_integration_impl.cpp
+)
+
+PEERDIR(
ydb/library/yql/providers/dq/interface
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp b/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp
index 4da4d13d03..0336760176 100644
--- a/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp
+++ b/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.cpp
@@ -1,47 +1,47 @@
-#include "yql_dq_integration_impl.h"
-
-namespace NYql {
-
-ui64 TDqIntegrationBase::Partition(const TDqSettings& config, size_t maxPartitions, const TExprNode& node,
- TVector<TString>& partitions, TString* clusterName, TExprContext& ctx, bool canFallback) {
- Y_UNUSED(config);
+#include "yql_dq_integration_impl.h"
+
+namespace NYql {
+
+ui64 TDqIntegrationBase::Partition(const TDqSettings& config, size_t maxPartitions, const TExprNode& node,
+ TVector<TString>& partitions, TString* clusterName, TExprContext& ctx, bool canFallback) {
+ Y_UNUSED(config);
Y_UNUSED(maxPartitions);
- Y_UNUSED(node);
- Y_UNUSED(partitions);
+ Y_UNUSED(node);
+ Y_UNUSED(partitions);
Y_UNUSED(clusterName);
Y_UNUSED(ctx);
- Y_UNUSED(canFallback);
+ Y_UNUSED(canFallback);
return 0;
-}
-
+}
+
TMaybe<ui64> TDqIntegrationBase::CanRead(const TDqSettings&, const TExprNode& read, TExprContext& ctx, bool skipIssues) {
- Y_UNUSED(read);
+ Y_UNUSED(read);
Y_UNUSED(ctx);
Y_UNUSED(skipIssues);
- return Nothing();
-}
-
+ return Nothing();
+}
+
TExprNode::TPtr TDqIntegrationBase::WrapRead(const TDqSettings&, const TExprNode::TPtr& read, TExprContext& ctx) {
- Y_UNUSED(read);
- Y_UNUSED(ctx);
-
- return read;
-}
-
-TMaybe<bool> TDqIntegrationBase::CanWrite(const TDqSettings&, const TExprNode& write, TExprContext& ctx) {
+ Y_UNUSED(read);
+ Y_UNUSED(ctx);
+
+ return read;
+}
+
+TMaybe<bool> TDqIntegrationBase::CanWrite(const TDqSettings&, const TExprNode& write, TExprContext& ctx) {
Y_UNUSED(write);
- Y_UNUSED(ctx);
+ Y_UNUSED(ctx);
return Nothing();
}
-void TDqIntegrationBase::RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) {
- Y_UNUSED(compiler);
-}
-
-bool TDqIntegrationBase::CanFallback() {
- return false;
-}
-
+void TDqIntegrationBase::RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) {
+ Y_UNUSED(compiler);
+}
+
+bool TDqIntegrationBase::CanFallback() {
+ return false;
+}
+
void TDqIntegrationBase::FillSourceSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sourceType) {
Y_UNUSED(node);
Y_UNUSED(settings);
@@ -54,23 +54,23 @@ void TDqIntegrationBase::FillSinkSettings(const TExprNode& node, ::google::proto
Y_UNUSED(sinkType);
}
-void TDqIntegrationBase::Annotate(const TExprNode& node, THashMap<TString, TString>& params) {
+void TDqIntegrationBase::Annotate(const TExprNode& node, THashMap<TString, TString>& params) {
Y_UNUSED(node);
- Y_UNUSED(params);
-}
-
-bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) {
- Y_UNUSED(root);
- Y_UNUSED(ctx);
- Y_UNUSED(params);
- Y_UNUSED(secureParams);
- return false;
-}
-
-void TDqIntegrationBase::WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) {
- Y_UNUSED(writer);
- Y_UNUSED(columns);
- Y_UNUSED(graphParams);
+ Y_UNUSED(params);
}
-} // namespace NYql
+bool TDqIntegrationBase::PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) {
+ Y_UNUSED(root);
+ Y_UNUSED(ctx);
+ Y_UNUSED(params);
+ Y_UNUSED(secureParams);
+ return false;
+}
+
+void TDqIntegrationBase::WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) {
+ Y_UNUSED(writer);
+ Y_UNUSED(columns);
+ Y_UNUSED(graphParams);
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.h b/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.h
index cd1c150ebf..824df83e38 100644
--- a/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.h
+++ b/ydb/library/yql/providers/common/dq/yql_dq_integration_impl.h
@@ -1,23 +1,23 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/providers/dq/interface/yql_dq_integration.h>
-
-namespace NYql {
-
-class TDqIntegrationBase: public IDqIntegration {
-public:
- ui64 Partition(const TDqSettings& config, size_t maxPartitions, const TExprNode& node,
- TVector<TString>& partitions, TString* clusterName, TExprContext& ctx, bool canFallback) override;
+
+namespace NYql {
+
+class TDqIntegrationBase: public IDqIntegration {
+public:
+ ui64 Partition(const TDqSettings& config, size_t maxPartitions, const TExprNode& node,
+ TVector<TString>& partitions, TString* clusterName, TExprContext& ctx, bool canFallback) override;
TMaybe<ui64> CanRead(const TDqSettings& config, const TExprNode& read, TExprContext& ctx, bool skipIssues) override;
TExprNode::TPtr WrapRead(const TDqSettings& config, const TExprNode::TPtr& read, TExprContext& ctx) override;
- void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) override;
- TMaybe<bool> CanWrite(const TDqSettings& config, const TExprNode& write, TExprContext& ctx) override;
- bool CanFallback() override;
+ void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) override;
+ TMaybe<bool> CanWrite(const TDqSettings& config, const TExprNode& write, TExprContext& ctx) override;
+ bool CanFallback() override;
void FillSourceSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sourceType) override;
void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) override;
- void Annotate(const TExprNode& node, THashMap<TString, TString>& params) override;
- bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) override;
- void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) override;
-};
-
-} // namespace NYql
+ void Annotate(const TExprNode& node, THashMap<TString, TString>& params) override;
+ bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) override;
+ void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) override;
+};
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/common/gateway/yql_provider_gateway.cpp b/ydb/library/yql/providers/common/gateway/yql_provider_gateway.cpp
index 3c92c84692..b2208e9f75 100644
--- a/ydb/library/yql/providers/common/gateway/yql_provider_gateway.cpp
+++ b/ydb/library/yql/providers/common/gateway/yql_provider_gateway.cpp
@@ -1,40 +1,40 @@
#include "yql_provider_gateway.h"
-
+
#include <ydb/library/yql/core/issue/yql_issue.h>
#include <ydb/library/yql/utils/log/log.h>
-
-namespace NYql {
-namespace NCommon {
-
+
+namespace NYql {
+namespace NCommon {
+
void TOperationResult::AddIssue(const TIssue& issue) {
WalkThroughIssues(issue, false, [&](const TIssue& err, ui16 level) {
Y_UNUSED(level);
YQL_CLOG(NOTICE, ProviderCommon) << err;
});
Issues_.AddIssue(issue);
-}
-
+}
+
void TOperationResult::AddIssues(const TIssues& issues) {
for (auto& topIssue: issues) {
WalkThroughIssues(topIssue, false, [&](const TIssue& err, ui16 level) {
Y_UNUSED(level);
YQL_CLOG(NOTICE, ProviderCommon) << err;
});
- }
+ }
Issues_.AddIssues(issues);
-}
-
+}
+
void TOperationResult::SetException(const std::exception& e, const TPosition& pos) {
- YQL_CLOG(ERROR, ProviderCommon) << e.what();
+ YQL_CLOG(ERROR, ProviderCommon) << e.what();
auto issue = ExceptionToIssue(e, pos);
Status_ = static_cast<NYql::EYqlIssueCode>(issue.GetCode());
Issues_.AddIssue(issue);
-}
-
+}
+
void TOperationResult::ReportIssues(TIssueManager& issueManager) const {
issueManager.RaiseIssues(Issues_);
}
-} // NCommon
-} // NYql
+} // NCommon
+} // NYql
diff --git a/ydb/library/yql/providers/common/gateway/yql_provider_gateway.h b/ydb/library/yql/providers/common/gateway/yql_provider_gateway.h
index c4b498e9a1..212e4b7fdc 100644
--- a/ydb/library/yql/providers/common/gateway/yql_provider_gateway.h
+++ b/ydb/library/yql/providers/common/gateway/yql_provider_gateway.h
@@ -29,9 +29,9 @@ private:
};
template<typename TResult>
-TResult ResultFromException(const std::exception& e, const TPosition& pos = {}) {
+TResult ResultFromException(const std::exception& e, const TPosition& pos = {}) {
TResult result;
- result.SetException(e, pos);
+ result.SetException(e, pos);
return result;
}
@@ -89,5 +89,5 @@ inline void SetPromiseValue(NThreading::TPromise<T>& promise, const NThreading::
});
}
-} // NCommon
-} // NYql
+} // NCommon
+} // NYql
diff --git a/ydb/library/yql/providers/common/metrics/metrics_registry.cpp b/ydb/library/yql/providers/common/metrics/metrics_registry.cpp
index 1fb7bdcbba..b1b46e161b 100644
--- a/ydb/library/yql/providers/common/metrics/metrics_registry.cpp
+++ b/ydb/library/yql/providers/common/metrics/metrics_registry.cpp
@@ -1,7 +1,7 @@
#include "metrics_registry.h"
#include <ydb/library/yql/providers/common/metrics/protos/metrics_registry.pb.h>
-
+
#include <util/generic/hash.h>
#include <util/generic/maybe.h>
#include <util/generic/stack.h>
diff --git a/ydb/library/yql/providers/common/metrics/metrics_registry.h b/ydb/library/yql/providers/common/metrics/metrics_registry.h
index d344ba9dd9..f9d27a0e09 100644
--- a/ydb/library/yql/providers/common/metrics/metrics_registry.h
+++ b/ydb/library/yql/providers/common/metrics/metrics_registry.h
@@ -6,10 +6,10 @@
namespace NYql {
-namespace NProto {
-class TMetricsRegistrySnapshot;
-}
-
+namespace NProto {
+class TMetricsRegistrySnapshot;
+}
+
struct IMetricsRegistry;
using IMetricsRegistryPtr = TIntrusivePtr<IMetricsRegistry>;
diff --git a/ydb/library/yql/providers/common/mkql/ya.make b/ydb/library/yql/providers/common/mkql/ya.make
index 8de943fad2..3bda9c2d74 100644
--- a/ydb/library/yql/providers/common/mkql/ya.make
+++ b/ydb/library/yql/providers/common/mkql/ya.make
@@ -11,8 +11,8 @@ SRCS(
parser.h
yql_provider_mkql.cpp
yql_provider_mkql.h
- yql_type_mkql.cpp
- yql_type_mkql.h
+ yql_type_mkql.cpp
+ yql_type_mkql.h
)
PEERDIR(
@@ -28,7 +28,7 @@ PEERDIR(
ydb/library/yql/providers/dq/expr_nodes
)
-YQL_LAST_ABI_VERSION()
+YQL_LAST_ABI_VERSION()
END()
diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
index dd1f162d78..01f78034bd 100644
--- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
+++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.cpp
@@ -1,5 +1,5 @@
#include "yql_provider_mkql.h"
-#include "yql_type_mkql.h"
+#include "yql_type_mkql.h"
#include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h>
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
@@ -234,22 +234,22 @@ void TMkqlCallableCompilerBase::AddCallable(const std::initializer_list<std::str
}
void TMkqlCallableCompilerBase::ChainCallable(const std::string_view& name, TCompiler compiler) {
- auto prevCompiler = GetCallable(name);
- auto chainedCompiler = [compiler = std::move(compiler), prevCompiler = std::move(prevCompiler)](const TExprNode& node, TMkqlBuildContext& ctx) -> NKikimr::NMiniKQL::TRuntimeNode {
- if (auto res = compiler(node, ctx)) {
- return res;
- }
- return prevCompiler(node, ctx);
- };
- OverrideCallable(name, chainedCompiler);
-}
-
+ auto prevCompiler = GetCallable(name);
+ auto chainedCompiler = [compiler = std::move(compiler), prevCompiler = std::move(prevCompiler)](const TExprNode& node, TMkqlBuildContext& ctx) -> NKikimr::NMiniKQL::TRuntimeNode {
+ if (auto res = compiler(node, ctx)) {
+ return res;
+ }
+ return prevCompiler(node, ctx);
+ };
+ OverrideCallable(name, chainedCompiler);
+}
+
void TMkqlCallableCompilerBase::ChainCallable(const std::initializer_list<std::string_view>& names, TCompiler compiler) {
- for (const auto& name : names) {
- ChainCallable(name, compiler);
- }
-}
-
+ for (const auto& name : names) {
+ ChainCallable(name, compiler);
+ }
+}
+
void TMkqlCallableCompilerBase::AddSimpleCallables(const std::initializer_list<std::pair<std::string_view, TProgramBuilder::UnaryFunctionMethod>>& callables) {
for (const auto& callable : callables) {
AddCallable(callable.first,
@@ -1483,9 +1483,9 @@ TMkqlCommonCallableCompiler::TShared::TShared() {
ui64 memLimit = 0U;
if (const auto memLimitSetting = GetSetting(*node.Child(6), "memLimit")) {
- memLimit = FromString<ui64>(memLimitSetting->Child(1)->Content());
- }
-
+ memLimit = FromString<ui64>(memLimitSetting->Child(1)->Content());
+ }
+
std::optional<ui32> sortedTableOrder;
if (const auto sortSetting = GetSetting(*node.Child(6), "sorted")) {
sortedTableOrder = sortSetting->Child(1)->Content() == "left" ? 0 : 1;
@@ -1507,47 +1507,47 @@ TMkqlCommonCallableCompiler::TShared::TShared() {
AddCallable("CombineCore", [](const TExprNode& node, TMkqlBuildContext& ctx) {
NNodes::TCoCombineCore core(&node);
-
+
const auto stream = MkqlBuildExpr(core.Input().Ref(), ctx);
const auto memLimit = FromString<ui64>(core.MemLimit().Cast().Value());
-
+
const auto keyExtractor = [&](TRuntimeNode item) {
return MkqlBuildLambda(core.KeyExtractor().Ref(), ctx, {item});
- };
+ };
const auto init = [&](TRuntimeNode key, TRuntimeNode item) {
return MkqlBuildLambda(core.InitHandler().Ref(), ctx, {key, item});
- };
+ };
const auto update = [&](TRuntimeNode key, TRuntimeNode item, TRuntimeNode state) {
return MkqlBuildLambda(core.UpdateHandler().Ref(), ctx, {key, item, state});
- };
+ };
const auto finish = [&](TRuntimeNode key, TRuntimeNode state) {
return MkqlBuildLambda(core.FinishHandler().Ref(), ctx, {key, state});
- };
-
+ };
+
return ctx.ProgramBuilder.CombineCore(stream, keyExtractor, init, update, finish, memLimit);
- });
-
+ });
+
AddCallable("GroupingCore", [](const TExprNode& node, TMkqlBuildContext& ctx) {
NNodes::TCoGroupingCore core(&node);
-
+
const auto stream = MkqlBuildExpr(core.Input().Ref(), ctx);
-
+
const auto groupSwitch = [&](TRuntimeNode key, TRuntimeNode item) {
return MkqlBuildLambda(core.GroupSwitch().Ref(), ctx, {key, item});
- };
+ };
const auto keyExtractor = [&](TRuntimeNode item) {
return MkqlBuildLambda(core.KeyExtractor().Ref(), ctx, {item});
- };
- TProgramBuilder::TUnaryLambda handler;
- if (auto lambda = core.ConvertHandler()) {
- handler = [&](TRuntimeNode item) {
- return MkqlBuildLambda(core.ConvertHandler().Ref(), ctx, {item});
- };
- }
-
- return ctx.ProgramBuilder.GroupingCore(stream, groupSwitch, keyExtractor, handler);
- });
-
+ };
+ TProgramBuilder::TUnaryLambda handler;
+ if (auto lambda = core.ConvertHandler()) {
+ handler = [&](TRuntimeNode item) {
+ return MkqlBuildLambda(core.ConvertHandler().Ref(), ctx, {item});
+ };
+ }
+
+ return ctx.ProgramBuilder.GroupingCore(stream, groupSwitch, keyExtractor, handler);
+ });
+
AddCallable("Chopper", [](const TExprNode& node, TMkqlBuildContext& ctx) {
const auto stream = MkqlBuildExpr(node.Head(), ctx);
@@ -1642,7 +1642,7 @@ TMkqlCommonCallableCompiler::TShared::TShared() {
const auto list = MkqlBuildExpr(node.Head(), ctx);
TMaybe<bool> isMany;
TMaybe<bool> isHashed;
- TMaybe<ui64> itemsCount;
+ TMaybe<ui64> itemsCount;
bool isCompact;
if (const auto error = ParseToDictSettings(node, ctx.ExprCtx, isMany, isHashed, itemsCount, isCompact)) {
ythrow TNodeException(node) << error->Message;
@@ -2105,7 +2105,7 @@ TMkqlCommonCallableCompiler::TShared::TShared() {
return ctx.ProgramBuilder.JoinDict(dict1, multi1, dict2, multi2, joinKind);
});
- AddCallable({"FilePath", "FileContent", "FolderPath"}, [](const TExprNode& node, TMkqlBuildContext& ctx) {
+ AddCallable({"FilePath", "FileContent", "FolderPath"}, [](const TExprNode& node, TMkqlBuildContext& ctx) {
TCallableBuilder call(ctx.ProgramBuilder.GetTypeEnvironment(), node.Content(), ctx.ProgramBuilder.NewDataType(NUdf::TDataType<char*>::Id));
call.Add(ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(node.Head().Content()));
return TRuntimeNode(call.Build(), false);
@@ -2277,7 +2277,7 @@ TMkqlCommonCallableCompiler::TShared::TShared() {
AddCallable("DependsOn", [](const TExprNode& node, TMkqlBuildContext& ctx) {
return MkqlBuildExpr(node.Head(), ctx);
});
-
+
AddCallable("Parameter", [](const TExprNode& node, TMkqlBuildContext& ctx) {
const NNodes::TCoParameter parameter(&node);
return ctx.ProgramBuilder.Member(ctx.Parameters, parameter.Name());
@@ -2286,37 +2286,37 @@ TMkqlCommonCallableCompiler::TShared::TShared() {
AddCallable("SecureParam", [](const TExprNode& node, TMkqlBuildContext& ctx) {
return ctx.ProgramBuilder.NewDataLiteral<NUdf::EDataSlot::String>(node.Head().Content());
});
-
+
AddCallable(SkippableCallables, [](const TExprNode& node, TMkqlBuildContext& ctx) {
return MkqlBuildExpr(node.Head(), ctx);
- });
-
- AddCallable("Merge", [](const TExprNode& node, TMkqlBuildContext& ctx) {
- const auto& args = GetAllArguments(node, ctx);
- auto extend = ctx.ProgramBuilder.Extend(args);
-
- if (auto sortConstr = node.GetConstraint<TSortedConstraintNode>()) {
+ });
+
+ AddCallable("Merge", [](const TExprNode& node, TMkqlBuildContext& ctx) {
+ const auto& args = GetAllArguments(node, ctx);
+ auto extend = ctx.ProgramBuilder.Extend(args);
+
+ if (auto sortConstr = node.GetConstraint<TSortedConstraintNode>()) {
const auto input = MkqlBuildExpr(node.Head(), ctx);
const auto& content = sortConstr->GetContent();
- std::vector<TRuntimeNode> ascending;
+ std::vector<TRuntimeNode> ascending;
ascending.reserve(content.size());
for (const auto& c: content) {
ascending.push_back(ctx.ProgramBuilder.NewDataLiteral(c.second));
- }
- TProgramBuilder::TUnaryLambda keyExractor = [&](TRuntimeNode item) {
- std::vector<TRuntimeNode> keys;
+ }
+ TProgramBuilder::TUnaryLambda keyExractor = [&](TRuntimeNode item) {
+ std::vector<TRuntimeNode> keys;
keys.reserve(content.size());
for (const auto& c : content) {
keys.push_back(ctx.ProgramBuilder.Member(item, c.first.front()));
- }
- return ctx.ProgramBuilder.NewTuple(keys);
- };
- return ctx.ProgramBuilder.Sort(extend, ctx.ProgramBuilder.NewTuple(ascending), keyExractor);
- }
- else {
- return extend;
- }
- });
+ }
+ return ctx.ProgramBuilder.NewTuple(keys);
+ };
+ return ctx.ProgramBuilder.Sort(extend, ctx.ProgramBuilder.NewTuple(ascending), keyExractor);
+ }
+ else {
+ return extend;
+ }
+ });
}
TRuntimeNode MkqlBuildLambda(const TExprNode& lambda, TMkqlBuildContext& ctx, const TRuntimeNode::TList& args) {
diff --git a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h
index a25c03b488..6cf3485ce8 100644
--- a/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h
+++ b/ydb/library/yql/providers/common/mkql/yql_provider_mkql.h
@@ -10,7 +10,7 @@ namespace NCommon {
class IMkqlCallableCompiler;
struct TMkqlBuildContext {
- using TArgumentsMap = TNodeMap<NKikimr::NMiniKQL::TRuntimeNode>;
+ using TArgumentsMap = TNodeMap<NKikimr::NMiniKQL::TRuntimeNode>;
using TMemoizedNodesMap = TArgumentsMap;
const IMkqlCallableCompiler& MkqlCompiler;
diff --git a/ydb/library/yql/providers/common/mkql/yql_type_mkql.cpp b/ydb/library/yql/providers/common/mkql/yql_type_mkql.cpp
index 4ba0e6d740..d8004ded58 100644
--- a/ydb/library/yql/providers/common/mkql/yql_type_mkql.cpp
+++ b/ydb/library/yql/providers/common/mkql/yql_type_mkql.cpp
@@ -1,17 +1,17 @@
-#include "yql_type_mkql.h"
+#include "yql_type_mkql.h"
#include <ydb/library/yql/minikql/mkql_program_builder.h>
#include <ydb/library/yql/minikql/mkql_type_builder.h>
#include <ydb/library/yql/minikql/mkql_runtime_version.h>
#include <ydb/library/yql/minikql/mkql_node_cast.h>
-
+
#include <ydb/library/yql/public/udf/udf_data_type.h>
#include <ydb/library/yql/public/udf/udf_type_inspection.h>
-
+
#include <ydb/library/yql/utils/yql_panic.h>
-#include <util/generic/vector.h>
-#include <util/string/cast.h>
+#include <util/generic/vector.h>
+#include <util/string/cast.h>
namespace NYql {
namespace NCommon {
@@ -22,8 +22,8 @@ NKikimr::NMiniKQL::TType* BuildType(const TTypeAnnotationNode& annotation, NKiki
auto data = annotation.Cast<TDataExprType>();
auto slot = data->GetSlot();
- const auto schemeType = NUdf::GetDataTypeInfo(slot).TypeId;
- if (NUdf::TDataType<NUdf::TDecimal>::Id == schemeType) {
+ const auto schemeType = NUdf::GetDataTypeInfo(slot).TypeId;
+ if (NUdf::TDataType<NUdf::TDecimal>::Id == schemeType) {
const auto params = static_cast<const TDataExprParamsType&>(annotation);
return pgmBuilder.NewDecimalType(FromString<ui8>(params.GetParamOne()), FromString<ui8>(params.GetParamTwo()));
} else {
@@ -34,16 +34,16 @@ NKikimr::NMiniKQL::TType* BuildType(const TTypeAnnotationNode& annotation, NKiki
case ETypeAnnotationKind::Struct: {
auto structObj = annotation.Cast<TStructExprType>();
std::vector<std::pair<std::string_view, NKikimr::NMiniKQL::TType*>> members;
- members.reserve(structObj->GetItems().size());
-
+ members.reserve(structObj->GetItems().size());
+
for (auto& item : structObj->GetItems()) {
auto itemType = BuildType(*item->GetItemType(), pgmBuilder, err, withTagged);
if (!itemType) {
return nullptr;
}
- members.emplace_back(item->GetName(), itemType);
+ members.emplace_back(item->GetName(), itemType);
}
- return pgmBuilder.NewStructType(members);
+ return pgmBuilder.NewStructType(members);
}
case ETypeAnnotationKind::List: {
@@ -67,7 +67,7 @@ NKikimr::NMiniKQL::TType* BuildType(const TTypeAnnotationNode& annotation, NKiki
case ETypeAnnotationKind::Tuple: {
auto tuple = annotation.Cast<TTupleExprType>();
TVector<NKikimr::NMiniKQL::TType*> elements;
- elements.reserve(tuple->GetItems().size());
+ elements.reserve(tuple->GetItems().size());
for (auto& child : tuple->GetItems()) {
elements.push_back(BuildType(*child, pgmBuilder, err, withTagged));
if (!elements.back()) {
@@ -203,19 +203,19 @@ NKikimr::NMiniKQL::TType* BuildType(const TTypeAnnotationNode& annotation, NKiki
}
}
-NKikimr::NMiniKQL::TType* BuildType(TPositionHandle pos, const TTypeAnnotationNode& annotation, NKikimr::NMiniKQL::TProgramBuilder& pgmBuilder, bool withTagged) {
+NKikimr::NMiniKQL::TType* BuildType(TPositionHandle pos, const TTypeAnnotationNode& annotation, NKikimr::NMiniKQL::TProgramBuilder& pgmBuilder, bool withTagged) {
TStringStream err;
auto type = BuildType(annotation, pgmBuilder, err, withTagged);
if (!type) {
- ythrow TNodeException(pos) << err.Str();
+ ythrow TNodeException(pos) << err.Str();
}
return type;
}
-NKikimr::NMiniKQL::TType* BuildType(const TExprNode& owner, const TTypeAnnotationNode& annotation, NKikimr::NMiniKQL::TProgramBuilder& pgmBuilder, bool withTagged) {
- return BuildType(owner.Pos(), annotation, pgmBuilder, withTagged);
-}
-
+NKikimr::NMiniKQL::TType* BuildType(const TExprNode& owner, const TTypeAnnotationNode& annotation, NKikimr::NMiniKQL::TProgramBuilder& pgmBuilder, bool withTagged) {
+ return BuildType(owner.Pos(), annotation, pgmBuilder, withTagged);
+}
+
const TTypeAnnotationNode* ConvertMiniKQLType(TPosition position, NKikimr::NMiniKQL::TType* type, TExprContext& ctx) {
using namespace NKikimr::NMiniKQL;
switch (type->GetKind()) {
@@ -228,7 +228,7 @@ const TTypeAnnotationNode* ConvertMiniKQLType(TPosition position, NKikimr::NMini
case TType::EKind::Data:
{
auto dataType = static_cast<TDataType*>(type);
- auto slot = NUdf::FindDataSlot(dataType->GetSchemeType());
+ auto slot = NUdf::FindDataSlot(dataType->GetSchemeType());
YQL_ENSURE(slot, "Unknown datatype: " << dataType->GetSchemeType());
if (*slot == EDataSlot::Decimal) {
const auto params = static_cast<TDataDecimalType*>(dataType)->GetParams();
diff --git a/ydb/library/yql/providers/common/mkql/yql_type_mkql.h b/ydb/library/yql/providers/common/mkql/yql_type_mkql.h
index 8f564c12ab..db7a7b49ba 100644
--- a/ydb/library/yql/providers/common/mkql/yql_type_mkql.h
+++ b/ydb/library/yql/providers/common/mkql/yql_type_mkql.h
@@ -17,7 +17,7 @@ namespace NYql {
namespace NCommon {
NKikimr::NMiniKQL::TType* BuildType(const TTypeAnnotationNode& annotation, NKikimr::NMiniKQL::TProgramBuilder& pgmBuilder, IOutputStream& err, bool withTagged = false);
-NKikimr::NMiniKQL::TType* BuildType(TPositionHandle pos, const TTypeAnnotationNode& annotation, NKikimr::NMiniKQL::TProgramBuilder& pgmBuilder, bool withTagged = false);
+NKikimr::NMiniKQL::TType* BuildType(TPositionHandle pos, const TTypeAnnotationNode& annotation, NKikimr::NMiniKQL::TProgramBuilder& pgmBuilder, bool withTagged = false);
NKikimr::NMiniKQL::TType* BuildType(const TExprNode& owner, const TTypeAnnotationNode& annotation, NKikimr::NMiniKQL::TProgramBuilder& pgmBuilder, bool withTagged = false);
const TTypeAnnotationNode* ConvertMiniKQLType(TPosition position, NKikimr::NMiniKQL::TType* type, TExprContext& ctx);
diff --git a/ydb/library/yql/providers/common/proto/gateways_config.proto b/ydb/library/yql/providers/common/proto/gateways_config.proto
index 8561fe2536..a1da4f226c 100644
--- a/ydb/library/yql/providers/common/proto/gateways_config.proto
+++ b/ydb/library/yql/providers/common/proto/gateways_config.proto
@@ -3,41 +3,41 @@ option java_package = "ru.yandex.yql.proto";
import "ydb/library/yql/protos/clickhouse.proto";
-/////////////////////////////// common ///////////////////////////////
-
-message TActivationByHour {
- required uint32 Hour = 1;
- required uint32 Percentage = 2;
-}
-
-message TActivationPercentage {
- optional uint32 Percentage = 1 [default = 0];
- repeated TActivationByHour ByHour = 2;
- repeated string IncludeUsers = 3;
- repeated string ExcludeUsers = 4;
- optional bool ExcludeRobots = 5 [default = true];
-}
-
+/////////////////////////////// common ///////////////////////////////
+
+message TActivationByHour {
+ required uint32 Hour = 1;
+ required uint32 Percentage = 2;
+}
+
+message TActivationPercentage {
+ optional uint32 Percentage = 1 [default = 0];
+ repeated TActivationByHour ByHour = 2;
+ repeated string IncludeUsers = 3;
+ repeated string ExcludeUsers = 4;
+ optional bool ExcludeRobots = 5 [default = true];
+}
+
message TAttr {
required string Name = 1;
required string Value = 2;
- optional TActivationPercentage Activation = 3;
-}
-
-message TRemoteFilePattern {
- required string Pattern = 1; // regexp to match
- required string Cluster = 2; // replacement string for YT server
- required string Path = 3; // replacement string for remote file path
-}
-
-enum EYtLogLevel {
- YL_NONE = -1;
- YL_FATAL = 0;
- YL_ERROR = 1;
- YL_INFO = 2;
- YL_DEBUG = 3;
-};
-
+ optional TActivationPercentage Activation = 3;
+}
+
+message TRemoteFilePattern {
+ required string Pattern = 1; // regexp to match
+ required string Cluster = 2; // replacement string for YT server
+ required string Path = 3; // replacement string for remote file path
+}
+
+enum EYtLogLevel {
+ YL_NONE = -1;
+ YL_FATAL = 0;
+ YL_ERROR = 1;
+ YL_INFO = 2;
+ YL_DEBUG = 3;
+};
+
enum EHostScheme {
HS_HTTP = 0;
HS_HTTPS = 1;
@@ -57,40 +57,40 @@ message THttpGatewayConfig {
optional uint64 MaxSimulatenousDownloadsSize = 2;
}
-/////////////////////////////// YT ///////////////////////////////
-
-message TYtClusterConfig {
+/////////////////////////////// YT ///////////////////////////////
+
+message TYtClusterConfig {
optional string Name = 1; // Short cluster name
optional string Cluster = 2; // Real cluster server name with port
- optional bool Default = 3 [default = false]; // Default cluster
- optional string YTToken = 4; // token to use at YT servers
- optional string YTName = 5; // YT cluster name
+ optional bool Default = 3 [default = false]; // Default cluster
+ optional string YTToken = 4; // token to use at YT servers
+ optional string YTName = 5; // YT cluster name
repeated TAttr Settings = 100;
}
-message TYtGatewayConfig {
- optional uint32 GatewayThreads = 1 [default = 0]; // Number of gateway MtpQueue threads. The adaptive MtpQueue will be used for 0 value
- optional EYtLogLevel YtLogLevel = 2 [default = YL_ERROR]; // The verbosity level of YT log
- optional string PerfToolsPath = 4; // Path to perftools libraries
- optional bool PerfToolsFromPackage = 5; // Use perftools from package
- optional string MrJobBin = 6; // Path to mrjob executable. If not set then main executable will be used
- optional string MrJobBinMd5 = 13; // MD5 checksum of mrjob executable. Should not be specified directly
- optional string MrJobUdfsDir = 7; // Path to linux udfs. If not set then loaded into main executable will be used
+message TYtGatewayConfig {
+ optional uint32 GatewayThreads = 1 [default = 0]; // Number of gateway MtpQueue threads. The adaptive MtpQueue will be used for 0 value
+ optional EYtLogLevel YtLogLevel = 2 [default = YL_ERROR]; // The verbosity level of YT log
+ optional string PerfToolsPath = 4; // Path to perftools libraries
+ optional bool PerfToolsFromPackage = 5; // Use perftools from package
+ optional string MrJobBin = 6; // Path to mrjob executable. If not set then main executable will be used
+ optional string MrJobBinMd5 = 13; // MD5 checksum of mrjob executable. Should not be specified directly
+ optional string MrJobUdfsDir = 7; // Path to linux udfs. If not set then loaded into main executable will be used
optional bool ExecuteUdfLocallyIfPossible = 8; // Turns on local execution mode for calc job with UDF.
optional bool LocalChainTest = 9 [default = false]; // Chain local execution jobs for test purpose.
- optional string YtDebugLogFile = 10; // File for full YT debug log
- optional uint64 YtDebugLogSize = 11 [default = 0]; // Max YT debug log size
- optional bool YtDebugLogAlwaysWrite = 12 [default = false]; // Write YT debug log always or on error only
+ optional string YtDebugLogFile = 10; // File for full YT debug log
+ optional uint64 YtDebugLogSize = 11 [default = 0]; // Max YT debug log size
+ optional bool YtDebugLogAlwaysWrite = 12 [default = false]; // Write YT debug log always or on error only
optional string LocalChainFile = 14; // File to dump table contet in test mode
- repeated TRemoteFilePattern RemoteFilePatterns = 100;
- repeated TYtClusterConfig ClusterMapping = 101; // Accepted cluster shortcuts. All names not from this list will be rejected
- repeated TAttr DefaultSettings = 102;
+ repeated TRemoteFilePattern RemoteFilePatterns = 100;
+ repeated TYtClusterConfig ClusterMapping = 101; // Accepted cluster shortcuts. All names not from this list will be rejected
+ repeated TAttr DefaultSettings = 102;
}
-/////////////////////////////// Kikimr ///////////////////////////////
-
+/////////////////////////////// Kikimr ///////////////////////////////
+
message THostPort {
required string Host = 1;
required uint32 Port = 2;
@@ -163,8 +163,8 @@ message TYdbGatewayConfig {
repeated TAttr DefaultSettings = 4;
}
-///////////////////////////// ClickHouse /////////////////////////////
-
+///////////////////////////// ClickHouse /////////////////////////////
+
message TClickHouseClusterConfig {
optional string Name = 1;
optional bool Default = 2 [default = false];
@@ -407,17 +407,17 @@ message TDqGatewayConfig {
repeated TAttr DefaultSettings = 102;
}
-/////////////////////////////// Yql Core ///////////////////////////////
-
-message TCoreAttr {
- required string Name = 1;
- repeated string Args = 2;
-}
-
-message TYqlCoreConfig {
- repeated TCoreAttr Flags = 1;
-}
-
+/////////////////////////////// Yql Core ///////////////////////////////
+
+message TCoreAttr {
+ required string Name = 1;
+ repeated string Args = 2;
+}
+
+message TYqlCoreConfig {
+ repeated TCoreAttr Flags = 1;
+}
+
/////////////////////////////// Sql Core ///////////////////////////////
message TWarnAsErrorByHourPercentage {
required uint32 Hour = 1;
@@ -429,23 +429,23 @@ message TSqlCoreConfig {
// warning to become an error
repeated TWarnAsErrorByHourPercentage V0SyntaxWarnAsErrorByHour = 2;
repeated string NoV0SyntaxErrorForUsers = 3;
-
- repeated string TranslationFlags = 4;
+
+ repeated string TranslationFlags = 4;
}
-/////////////////////////////// Root ///////////////////////////////
-
+/////////////////////////////// Root ///////////////////////////////
+
message TGatewaysConfig {
- optional TYtGatewayConfig Yt = 1;
- optional TKikimrGatewayConfig Kikimr = 2;
- optional TClickHouseGatewayConfig ClickHouse = 3;
+ optional TYtGatewayConfig Yt = 1;
+ optional TKikimrGatewayConfig Kikimr = 2;
+ optional TClickHouseGatewayConfig ClickHouse = 3;
optional TRtmrGatewayConfig Rtmr = 4;
optional TKikimrMvpGatewayConfig KikimrMvp = 5;
optional TStatGatewayConfig Stat = 6;
optional TChytGatewayConfig Chyt = 7;
optional TSolomonGatewayConfig Solomon = 8;
optional TFileStorageAdditionalConfig Fs = 9;
- optional TYqlCoreConfig YqlCore = 10;
+ optional TYqlCoreConfig YqlCore = 10;
optional TPostgresqlGatewayConfig Postgresql = 11;
optional TSqlCoreConfig SqlCore = 12;
optional TDqGatewayConfig Dq = 13;
diff --git a/ydb/library/yql/providers/common/proto/udf_resolver.proto b/ydb/library/yql/providers/common/proto/udf_resolver.proto
index 170d13e6d7..f517406421 100644
--- a/ydb/library/yql/providers/common/proto/udf_resolver.proto
+++ b/ydb/library/yql/providers/common/proto/udf_resolver.proto
@@ -1,12 +1,12 @@
-package NYql;
+package NYql;
option java_package = "ru.yandex.yql.proto";
-
-message TFunction {
- required string Name = 1;
- required string TypeConfig = 2;
- optional string UserType = 3;
-};
-
+
+message TFunction {
+ required string Name = 1;
+ required string TypeConfig = 2;
+ optional string UserType = 3;
+};
+
message TArgResult {
required string Name = 1;
required string Type = 2;
@@ -14,11 +14,11 @@ message TArgResult {
required uint32 Flags = 4;
};
-message TFunctionResult {
- optional string NormalizedUserType = 1;
- optional string RunConfigType = 2;
- optional string CallableType = 3;
- optional string Error = 4;
+message TFunctionResult {
+ optional string NormalizedUserType = 1;
+ optional string RunConfigType = 2;
+ optional string CallableType = 3;
+ optional string Error = 4;
optional string Name = 5;
optional string Doc = 7;
optional bool IsTypeAwareness = 8;
@@ -27,28 +27,28 @@ message TFunctionResult {
repeated TArgResult Args = 11;
optional string ReturnType = 12;
optional string ReturnDoc = 13;
-};
-
-message TImport {
- required string FileAlias = 1;
- required string Path = 2;
- optional bool System = 3 [default = false];
-};
-
-message TImportResult {
- required string FileAlias = 1;
- repeated string Modules = 2;
- optional string Error = 3;
+};
+
+message TImport {
+ required string FileAlias = 1;
+ required string Path = 2;
+ optional bool System = 3 [default = false];
+};
+
+message TImportResult {
+ required string FileAlias = 1;
+ repeated string Modules = 2;
+ optional string Error = 3;
optional uint64 FileSize = 4;
optional string Md5 = 5;
-};
-
-message TResolve {
- repeated TImport Imports = 1;
- repeated TFunction Udfs = 2;
-};
-
-message TResolveResult {
- repeated TImportResult Imports = 1;
- repeated TFunctionResult Udfs = 2;
-};
+};
+
+message TResolve {
+ repeated TImport Imports = 1;
+ repeated TFunction Udfs = 2;
+};
+
+message TResolveResult {
+ repeated TImportResult Imports = 1;
+ repeated TFunctionResult Udfs = 2;
+};
diff --git a/ydb/library/yql/providers/common/proto/ya.make b/ydb/library/yql/providers/common/proto/ya.make
index 5b79b7dd12..45332f22d8 100644
--- a/ydb/library/yql/providers/common/proto/ya.make
+++ b/ydb/library/yql/providers/common/proto/ya.make
@@ -4,7 +4,7 @@ PROTO_LIBRARY()
SRCS(
gateways_config.proto
- udf_resolver.proto
+ udf_resolver.proto
)
PEERDIR(
diff --git a/ydb/library/yql/providers/common/provider/ya.make b/ydb/library/yql/providers/common/provider/ya.make
index 0d927b3376..1e36b5957e 100644
--- a/ydb/library/yql/providers/common/provider/ya.make
+++ b/ydb/library/yql/providers/common/provider/ya.make
@@ -7,8 +7,8 @@ OWNER(
)
SRCS(
- yql_data_provider_impl.cpp
- yql_data_provider_impl.h
+ yql_data_provider_impl.cpp
+ yql_data_provider_impl.h
yql_provider.cpp
yql_provider.h
yql_provider_names.h
diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp
index 8de3d57fd3..53e06dc63d 100644
--- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp
+++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.cpp
@@ -1,63 +1,63 @@
-#include "yql_data_provider_impl.h"
-
+#include "yql_data_provider_impl.h"
+
#include <ydb/library/yql/core/yql_expr_constraint.h>
-
-#include <util/system/compiler.h>
-
-namespace NYql {
-
+
+#include <util/system/compiler.h>
+
+namespace NYql {
+
void TPlanFormatterBase::WriteDetails(const TExprNode& node, NYson::TYsonWriter& writer) {
- Y_UNUSED(node);
- Y_UNUSED(writer);
-}
-
-bool TPlanFormatterBase::GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) {
- Y_UNUSED(node);
- Y_UNUSED(children);
- Y_UNUSED(compact);
- return false;
-}
-
-void TPlanFormatterBase::GetResultDependencies(const TExprNode::TPtr& node, TExprNode::TListType& children, bool compact) {
- Y_UNUSED(node);
- Y_UNUSED(children);
- Y_UNUSED(compact);
-}
-
-void TPlanFormatterBase::GetInputs(const TExprNode& node, TVector<TPinInfo>& inputs) {
- Y_UNUSED(node);
- Y_UNUSED(inputs);
-}
-
-void TPlanFormatterBase::GetOutputs(const TExprNode& node, TVector<TPinInfo>& outputs) {
- Y_UNUSED(node);
- Y_UNUSED(outputs);
-}
-
-TString TPlanFormatterBase::GetProviderPath(const TExprNode& node) {
- Y_UNUSED(node);
- return TString();
-}
-
+ Y_UNUSED(node);
+ Y_UNUSED(writer);
+}
+
+bool TPlanFormatterBase::GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) {
+ Y_UNUSED(node);
+ Y_UNUSED(children);
+ Y_UNUSED(compact);
+ return false;
+}
+
+void TPlanFormatterBase::GetResultDependencies(const TExprNode::TPtr& node, TExprNode::TListType& children, bool compact) {
+ Y_UNUSED(node);
+ Y_UNUSED(children);
+ Y_UNUSED(compact);
+}
+
+void TPlanFormatterBase::GetInputs(const TExprNode& node, TVector<TPinInfo>& inputs) {
+ Y_UNUSED(node);
+ Y_UNUSED(inputs);
+}
+
+void TPlanFormatterBase::GetOutputs(const TExprNode& node, TVector<TPinInfo>& outputs) {
+ Y_UNUSED(node);
+ Y_UNUSED(outputs);
+}
+
+TString TPlanFormatterBase::GetProviderPath(const TExprNode& node) {
+ Y_UNUSED(node);
+ return TString();
+}
+
void TPlanFormatterBase::WritePlanDetails(const TExprNode& node, NYson::TYsonWriter& writer) {
- Y_UNUSED(node);
- Y_UNUSED(writer);
-}
-
+ Y_UNUSED(node);
+ Y_UNUSED(writer);
+}
+
void TPlanFormatterBase::WritePullDetails(const TExprNode& node, NYson::TYsonWriter& writer) {
- Y_UNUSED(node);
- Y_UNUSED(writer);
-}
-
+ Y_UNUSED(node);
+ Y_UNUSED(writer);
+}
+
void TPlanFormatterBase::WritePinDetails(const TExprNode& node, NYson::TYsonWriter& writer) {
- Y_UNUSED(node);
- Y_UNUSED(writer);
-}
-
-TString TPlanFormatterBase::GetOperationDisplayName(const TExprNode& node) {
- return TString(node.Content());
-}
-
+ Y_UNUSED(node);
+ Y_UNUSED(writer);
+}
+
+TString TPlanFormatterBase::GetOperationDisplayName(const TExprNode& node) {
+ return TString(node.Content());
+}
+
void TTrackableNodeProcessorBase::GetUsedNodes(const TExprNode& node, TVector<TString>& usedNodeIds) {
Y_UNUSED(node);
usedNodeIds.clear();
@@ -73,15 +73,15 @@ IGraphTransformer& TTrackableNodeProcessorBase::GetCleanupTransformer() {
return NullTransformer_;
}
-bool TDataProviderBase::Initialize(TExprContext& ctx) {
- Y_UNUSED(ctx);
- return true;
-}
-
-IGraphTransformer& TDataProviderBase::GetConfigurationTransformer() {
- return NullTransformer_;
-}
-
+bool TDataProviderBase::Initialize(TExprContext& ctx) {
+ Y_UNUSED(ctx);
+ return true;
+}
+
+IGraphTransformer& TDataProviderBase::GetConfigurationTransformer() {
+ return NullTransformer_;
+}
+
TExprNode::TPtr TDataProviderBase::GetClusterInfo(const TString& cluster, TExprContext& ctx) {
Y_UNUSED(cluster);
Y_UNUSED(ctx);
@@ -92,57 +92,57 @@ const THashMap<TString, TString>* TDataProviderBase::GetClusterTokens() {
return nullptr;
}
-IGraphTransformer& TDataProviderBase::GetIODiscoveryTransformer() {
- return NullTransformer_;
-}
-
-IGraphTransformer& TDataProviderBase::GetEpochsTransformer() {
- return NullTransformer_;
-}
-
-IGraphTransformer& TDataProviderBase::GetIntentDeterminationTransformer() {
- return NullTransformer_;
-}
-
-bool TDataProviderBase::ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) {
- Y_UNUSED(node);
- Y_UNUSED(ctx);
-
- cluster = Nothing();
- return true;
-}
-
-bool TDataProviderBase::CanParse(const TExprNode& node) {
- Y_UNUSED(node);
- return false;
-}
-
-void TDataProviderBase::FillModifyCallables(THashSet<TStringBuf>& callables) {
- Y_UNUSED(callables);
-}
-
-IGraphTransformer& TDataProviderBase::GetTypeAnnotationTransformer(bool instantOnly) {
- Y_UNUSED(instantOnly);
- return NullTransformer_;
-}
-
-IGraphTransformer& TDataProviderBase::GetConstraintTransformer(bool instantOnly, bool subGraph) {
- Y_UNUSED(instantOnly);
- Y_UNUSED(subGraph);
- if (!DefConstraintTransformer_) {
- DefConstraintTransformer_ = CreateDefCallableConstraintTransformer();
- }
- return *DefConstraintTransformer_;
-}
-
-TExprNode::TPtr TDataProviderBase::RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) {
- Y_UNUSED(ctx);
- return node;
-}
-
-void TDataProviderBase::PostRewriteIO() {
-}
-
+IGraphTransformer& TDataProviderBase::GetIODiscoveryTransformer() {
+ return NullTransformer_;
+}
+
+IGraphTransformer& TDataProviderBase::GetEpochsTransformer() {
+ return NullTransformer_;
+}
+
+IGraphTransformer& TDataProviderBase::GetIntentDeterminationTransformer() {
+ return NullTransformer_;
+}
+
+bool TDataProviderBase::ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) {
+ Y_UNUSED(node);
+ Y_UNUSED(ctx);
+
+ cluster = Nothing();
+ return true;
+}
+
+bool TDataProviderBase::CanParse(const TExprNode& node) {
+ Y_UNUSED(node);
+ return false;
+}
+
+void TDataProviderBase::FillModifyCallables(THashSet<TStringBuf>& callables) {
+ Y_UNUSED(callables);
+}
+
+IGraphTransformer& TDataProviderBase::GetTypeAnnotationTransformer(bool instantOnly) {
+ Y_UNUSED(instantOnly);
+ return NullTransformer_;
+}
+
+IGraphTransformer& TDataProviderBase::GetConstraintTransformer(bool instantOnly, bool subGraph) {
+ Y_UNUSED(instantOnly);
+ Y_UNUSED(subGraph);
+ if (!DefConstraintTransformer_) {
+ DefConstraintTransformer_ = CreateDefCallableConstraintTransformer();
+ }
+ return *DefConstraintTransformer_;
+}
+
+TExprNode::TPtr TDataProviderBase::RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) {
+ Y_UNUSED(ctx);
+ return node;
+}
+
+void TDataProviderBase::PostRewriteIO() {
+}
+
void TDataProviderBase::Reset() {
GetConfigurationTransformer().Rewind();
GetIODiscoveryTransformer().Rewind();
@@ -164,47 +164,47 @@ void TDataProviderBase::Reset() {
GetCallableExecutionTransformer().Rewind();
GetFinalizingTransformer().Rewind();
GetPlanInfoTransformer().Rewind();
- GetTrackableNodeProcessor().GetCleanupTransformer().Rewind();
-}
-
-IGraphTransformer& TDataProviderBase::GetRecaptureOptProposalTransformer() {
- return NullTransformer_;
-}
-
-IGraphTransformer& TDataProviderBase::GetLogicalOptProposalTransformer() {
- return NullTransformer_;
-}
-
-IGraphTransformer& TDataProviderBase::GetPhysicalOptProposalTransformer() {
- return NullTransformer_;
-}
-
-IGraphTransformer& TDataProviderBase::GetPhysicalFinalizingTransformer() {
- return NullTransformer_;
-}
-
-IGraphTransformer& TDataProviderBase::GetLoadTableMetadataTransformer() {
- return NullTransformer_;
-}
-
-bool TDataProviderBase::IsPersistent(const TExprNode& node) {
- Y_UNUSED(node);
- return false;
-}
-
-bool TDataProviderBase::CanBuildResult(const TExprNode& node, TSyncMap& syncList) {
- Y_UNUSED(node);
- Y_UNUSED(syncList);
- return false;
-}
-
-bool TDataProviderBase::CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) {
- Y_UNUSED(node);
- Y_UNUSED(syncList);
- Y_UNUSED(canRef);
- return false;
-}
-
+ GetTrackableNodeProcessor().GetCleanupTransformer().Rewind();
+}
+
+IGraphTransformer& TDataProviderBase::GetRecaptureOptProposalTransformer() {
+ return NullTransformer_;
+}
+
+IGraphTransformer& TDataProviderBase::GetLogicalOptProposalTransformer() {
+ return NullTransformer_;
+}
+
+IGraphTransformer& TDataProviderBase::GetPhysicalOptProposalTransformer() {
+ return NullTransformer_;
+}
+
+IGraphTransformer& TDataProviderBase::GetPhysicalFinalizingTransformer() {
+ return NullTransformer_;
+}
+
+IGraphTransformer& TDataProviderBase::GetLoadTableMetadataTransformer() {
+ return NullTransformer_;
+}
+
+bool TDataProviderBase::IsPersistent(const TExprNode& node) {
+ Y_UNUSED(node);
+ return false;
+}
+
+bool TDataProviderBase::CanBuildResult(const TExprNode& node, TSyncMap& syncList) {
+ Y_UNUSED(node);
+ Y_UNUSED(syncList);
+ return false;
+}
+
+bool TDataProviderBase::CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) {
+ Y_UNUSED(node);
+ Y_UNUSED(syncList);
+ Y_UNUSED(canRef);
+ return false;
+}
+
bool TDataProviderBase::GetExecWorld(const TExprNode& node, TExprNode::TPtr& root) {
root = nullptr;
Y_UNUSED(node);
@@ -221,63 +221,63 @@ void TDataProviderBase::UndoEvaluationChanges() {
GetEpochsTransformer().Rewind();
}
-TExprNode::TPtr TDataProviderBase::CleanupWorld(const TExprNode::TPtr& node, TExprContext& ctx) {
- Y_UNUSED(ctx);
- return node;
-}
-
-TExprNode::TPtr TDataProviderBase::OptimizePull(const TExprNode::TPtr& source, const TFillSettings& fillSettings,
- TExprContext& ctx, IOptimizationContext& optCtx)
-{
- Y_UNUSED(fillSettings);
- Y_UNUSED(ctx);
- Y_UNUSED(optCtx);
- return source;
-
-}
-
-bool TDataProviderBase::CanExecute(const TExprNode& node) {
- Y_UNUSED(node);
- return false;
-}
-
-void TDataProviderBase::GetRequiredChildren(const TExprNode& node, TExprNode::TListType& children) {
- TPlanFormatterBase::GetDependencies(node, children, false);
-}
-
-IGraphTransformer& TDataProviderBase::GetCallableExecutionTransformer() {
- return NullTransformer_;
-}
-
-IGraphTransformer& TDataProviderBase::GetFinalizingTransformer() {
- return NullTransformer_;
-}
-
+TExprNode::TPtr TDataProviderBase::CleanupWorld(const TExprNode::TPtr& node, TExprContext& ctx) {
+ Y_UNUSED(ctx);
+ return node;
+}
+
+TExprNode::TPtr TDataProviderBase::OptimizePull(const TExprNode::TPtr& source, const TFillSettings& fillSettings,
+ TExprContext& ctx, IOptimizationContext& optCtx)
+{
+ Y_UNUSED(fillSettings);
+ Y_UNUSED(ctx);
+ Y_UNUSED(optCtx);
+ return source;
+
+}
+
+bool TDataProviderBase::CanExecute(const TExprNode& node) {
+ Y_UNUSED(node);
+ return false;
+}
+
+void TDataProviderBase::GetRequiredChildren(const TExprNode& node, TExprNode::TListType& children) {
+ TPlanFormatterBase::GetDependencies(node, children, false);
+}
+
+IGraphTransformer& TDataProviderBase::GetCallableExecutionTransformer() {
+ return NullTransformer_;
+}
+
+IGraphTransformer& TDataProviderBase::GetFinalizingTransformer() {
+ return NullTransformer_;
+}
+
bool TDataProviderBase::CollectDiagnostics(NYson::TYsonWriter& writer) {
- Y_UNUSED(writer);
- return false;
-}
-
+ Y_UNUSED(writer);
+ return false;
+}
+
bool TDataProviderBase::GetTasksInfo(NYson::TYsonWriter& writer) {
- Y_UNUSED(writer);
- return false;
-}
-
+ Y_UNUSED(writer);
+ return false;
+}
+
bool TDataProviderBase::CollectStatistics(NYson::TYsonWriter& writer, bool totalOnly) {
Y_UNUSED(writer);
- Y_UNUSED(totalOnly);
+ Y_UNUSED(totalOnly);
return false;
}
bool TDataProviderBase::CollectDiscoveredData(NYson::TYsonWriter& writer) {
- Y_UNUSED(writer);
- return false;
-}
-
-IPlanFormatter& TDataProviderBase::GetPlanFormatter() {
- return *this;
-}
-
+ Y_UNUSED(writer);
+ return false;
+}
+
+IPlanFormatter& TDataProviderBase::GetPlanFormatter() {
+ return *this;
+}
+
ITrackableNodeProcessor& TDataProviderBase::GetTrackableNodeProcessor() {
return NullTrackableNodeProcessor_;
}
@@ -286,8 +286,8 @@ IGraphTransformer& TDataProviderBase::GetPlanInfoTransformer() {
return NullTransformer_;
}
-IDqIntegration* TDataProviderBase::GetDqIntegration() {
- return nullptr;
-}
-
-} // namespace NYql
+IDqIntegration* TDataProviderBase::GetDqIntegration() {
+ return nullptr;
+}
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h
index 242a39877e..a4ea6266a1 100644
--- a/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h
+++ b/ydb/library/yql/providers/common/provider/yql_data_provider_impl.h
@@ -1,29 +1,29 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/core/yql_data_provider.h>
#include <ydb/library/yql/core/yql_graph_transformer.h>
-
-#include <util/generic/ptr.h>
-
-namespace NYql {
-
-class TPlanFormatterBase : public IPlanFormatter {
-public:
- TPlanFormatterBase() = default;
- ~TPlanFormatterBase() = default;
-
+
+#include <util/generic/ptr.h>
+
+namespace NYql {
+
+class TPlanFormatterBase : public IPlanFormatter {
+public:
+ TPlanFormatterBase() = default;
+ ~TPlanFormatterBase() = default;
+
void WriteDetails(const TExprNode& node, NYson::TYsonWriter& writer) override;
- bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override;
- void GetResultDependencies(const TExprNode::TPtr& node, TExprNode::TListType& children, bool compact) override;
- void GetInputs(const TExprNode& node, TVector<TPinInfo>& inputs) override;
- void GetOutputs(const TExprNode& node, TVector<TPinInfo>& outputs) override;
- TString GetProviderPath(const TExprNode& node) override;
+ bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override;
+ void GetResultDependencies(const TExprNode::TPtr& node, TExprNode::TListType& children, bool compact) override;
+ void GetInputs(const TExprNode& node, TVector<TPinInfo>& inputs) override;
+ void GetOutputs(const TExprNode& node, TVector<TPinInfo>& outputs) override;
+ TString GetProviderPath(const TExprNode& node) override;
void WritePlanDetails(const TExprNode& node, NYson::TYsonWriter& writer) override;
void WritePullDetails(const TExprNode& node, NYson::TYsonWriter& writer) override;
void WritePinDetails(const TExprNode& node, NYson::TYsonWriter& writer) override;
- TString GetOperationDisplayName(const TExprNode& node) override;
-};
-
+ TString GetOperationDisplayName(const TExprNode& node) override;
+};
+
class TTrackableNodeProcessorBase : public ITrackableNodeProcessor {
public:
TTrackableNodeProcessorBase() = default;
@@ -36,57 +36,57 @@ protected:
TNullTransformer NullTransformer_;
};
-class TDataProviderBase : public IDataProvider, public TPlanFormatterBase {
-public:
- TDataProviderBase() = default;
- ~TDataProviderBase() = default;
-
- bool Initialize(TExprContext& ctx) override;
- IGraphTransformer& GetConfigurationTransformer() override;
+class TDataProviderBase : public IDataProvider, public TPlanFormatterBase {
+public:
+ TDataProviderBase() = default;
+ ~TDataProviderBase() = default;
+
+ bool Initialize(TExprContext& ctx) override;
+ IGraphTransformer& GetConfigurationTransformer() override;
TExprNode::TPtr GetClusterInfo(const TString& cluster, TExprContext& ctx) override;
const THashMap<TString, TString>* GetClusterTokens() override;
- IGraphTransformer& GetIODiscoveryTransformer() override;
- IGraphTransformer& GetEpochsTransformer() override;
- IGraphTransformer& GetIntentDeterminationTransformer() override;
- bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override;
- bool CanParse(const TExprNode& node) override;
- void FillModifyCallables(THashSet<TStringBuf>& callables) override;
- IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override;
- IGraphTransformer& GetConstraintTransformer(bool instantOnly, bool subGraph) override;
- TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override;
- void PostRewriteIO() override;
+ IGraphTransformer& GetIODiscoveryTransformer() override;
+ IGraphTransformer& GetEpochsTransformer() override;
+ IGraphTransformer& GetIntentDeterminationTransformer() override;
+ bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override;
+ bool CanParse(const TExprNode& node) override;
+ void FillModifyCallables(THashSet<TStringBuf>& callables) override;
+ IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override;
+ IGraphTransformer& GetConstraintTransformer(bool instantOnly, bool subGraph) override;
+ TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override;
+ void PostRewriteIO() override;
void Reset() override;
- IGraphTransformer& GetRecaptureOptProposalTransformer() override;
- IGraphTransformer& GetLogicalOptProposalTransformer() override;
- IGraphTransformer& GetPhysicalOptProposalTransformer() override;
- IGraphTransformer& GetPhysicalFinalizingTransformer() override;
- IGraphTransformer& GetLoadTableMetadataTransformer() override;
- bool IsPersistent(const TExprNode& node) override;
- bool CanBuildResult(const TExprNode& node, TSyncMap& syncList) override;
- bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) override;
+ IGraphTransformer& GetRecaptureOptProposalTransformer() override;
+ IGraphTransformer& GetLogicalOptProposalTransformer() override;
+ IGraphTransformer& GetPhysicalOptProposalTransformer() override;
+ IGraphTransformer& GetPhysicalFinalizingTransformer() override;
+ IGraphTransformer& GetLoadTableMetadataTransformer() override;
+ bool IsPersistent(const TExprNode& node) override;
+ bool CanBuildResult(const TExprNode& node, TSyncMap& syncList) override;
+ bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) override;
bool GetExecWorld(const TExprNode& node, TExprNode::TPtr& root) override;
bool CanEvaluate(const TExprNode& node) override;
void UndoEvaluationChanges() override;
- TExprNode::TPtr CleanupWorld(const TExprNode::TPtr& node, TExprContext& ctx) override;
- TExprNode::TPtr OptimizePull(const TExprNode::TPtr& source, const TFillSettings& fillSettings, TExprContext& ctx,
- IOptimizationContext& optCtx) override;
- bool CanExecute(const TExprNode& node) override;
- void GetRequiredChildren(const TExprNode& node, TExprNode::TListType& children) override;
- IGraphTransformer& GetCallableExecutionTransformer() override;
- IGraphTransformer& GetFinalizingTransformer() override;
+ TExprNode::TPtr CleanupWorld(const TExprNode::TPtr& node, TExprContext& ctx) override;
+ TExprNode::TPtr OptimizePull(const TExprNode::TPtr& source, const TFillSettings& fillSettings, TExprContext& ctx,
+ IOptimizationContext& optCtx) override;
+ bool CanExecute(const TExprNode& node) override;
+ void GetRequiredChildren(const TExprNode& node, TExprNode::TListType& children) override;
+ IGraphTransformer& GetCallableExecutionTransformer() override;
+ IGraphTransformer& GetFinalizingTransformer() override;
bool CollectDiagnostics(NYson::TYsonWriter& writer) override;
bool GetTasksInfo(NYson::TYsonWriter& writer) override;
bool CollectStatistics(NYson::TYsonWriter& writer, bool totalOnly) override;
bool CollectDiscoveredData(NYson::TYsonWriter& writer) override;
- IPlanFormatter& GetPlanFormatter() override;
+ IPlanFormatter& GetPlanFormatter() override;
ITrackableNodeProcessor& GetTrackableNodeProcessor() override;
IGraphTransformer& GetPlanInfoTransformer() override;
- IDqIntegration* GetDqIntegration() override;
-
-protected:
- THolder<IGraphTransformer> DefConstraintTransformer_;
- TNullTransformer NullTransformer_;
+ IDqIntegration* GetDqIntegration() override;
+
+protected:
+ THolder<IGraphTransformer> DefConstraintTransformer_;
+ TNullTransformer NullTransformer_;
TTrackableNodeProcessorBase NullTrackableNodeProcessor_;
-};
-
-} // namespace NYql
+};
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/common/provider/yql_provider.cpp b/ydb/library/yql/providers/common/provider/yql_provider.cpp
index aa782ee93f..3dc87e4202 100644
--- a/ydb/library/yql/providers/common/provider/yql_provider.cpp
+++ b/ydb/library/yql/providers/common/provider/yql_provider.cpp
@@ -10,9 +10,9 @@
#include <ydb/library/yql/minikql/mkql_function_registry.h>
#include <ydb/library/yql/minikql/mkql_program_builder.h>
-#include <util/folder/path.h>
-#include <util/generic/utility.h>
-
+#include <util/folder/path.h>
+#include <util/generic/utility.h>
+
namespace NYql {
namespace NCommon {
@@ -48,13 +48,13 @@ bool TCommitSettings::EnsureOtherEmpty(TExprContext& ctx) {
return true;
}
-TCoNameValueTupleList TCommitSettings::BuildNode(TExprContext& ctx) const {
+TCoNameValueTupleList TCommitSettings::BuildNode(TExprContext& ctx) const {
TVector<TExprBase> settings;
auto addSettings = [this, &settings, &ctx] (const TString& name, TMaybeNode<TExprBase> value) {
if (value) {
- auto node = Build<TCoNameValueTuple>(ctx, Pos)
+ auto node = Build<TCoNameValueTuple>(ctx, Pos)
.Name().Build(name)
.Value(value.Cast())
.Done();
@@ -71,7 +71,7 @@ TCoNameValueTupleList TCommitSettings::BuildNode(TExprContext& ctx) const {
settings.push_back(setting);
}
- auto ret = Build<TCoNameValueTupleList>(ctx, Pos)
+ auto ret = Build<TCoNameValueTupleList>(ctx, Pos)
.Add(settings)
.Done();
@@ -94,7 +94,7 @@ TExprNode::TPtr BuildTypeExpr(TPositionHandle pos, const TTypeAnnotationNode& an
return ExpandType(pos, ann, ctx);
}
-bool HasResOrPullOption(const TExprNode& node, const TStringBuf& option) {
+bool HasResOrPullOption(const TExprNode& node, const TStringBuf& option) {
if (node.Content() == "Result" || node.Content() == "Pull") {
auto options = node.Child(4);
for (auto setting : options->Children()) {
@@ -149,33 +149,33 @@ NYson::EYsonFormat GetYsonFormat(const IDataProvider::TFillSettings& fillSetting
}
TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) {
- TMaybeNode<TCoAtom> mode;
+ TMaybeNode<TCoAtom> mode;
TMaybeNode<TExprList> columns;
- TMaybeNode<TCoAtomList> primaryKey;
+ TMaybeNode<TCoAtomList> primaryKey;
TMaybeNode<TCoAtomList> partitionBy;
TMaybeNode<TCoNameValueTupleList> orderBy;
- TMaybeNode<TCoLambda> filter;
- TMaybeNode<TCoLambda> update;
- TVector<TCoNameValueTuple> other;
+ TMaybeNode<TCoLambda> filter;
+ TMaybeNode<TCoLambda> update;
+ TVector<TCoNameValueTuple> other;
TVector<TCoIndex> indexes;
TVector<TCoChangefeed> changefeeds;
TMaybeNode<TExprList> columnFamilies;
TVector<TCoNameValueTuple> tableSettings;
TVector<TCoNameValueTuple> alterActions;
for (auto child : node) {
- if (auto maybeTuple = child.Maybe<TCoNameValueTuple>()) {
+ if (auto maybeTuple = child.Maybe<TCoNameValueTuple>()) {
auto tuple = maybeTuple.Cast();
auto name = tuple.Name().Value();
if (name == "mode") {
- YQL_ENSURE(tuple.Value().Maybe<TCoAtom>());
- mode = tuple.Value().Cast<TCoAtom>();
+ YQL_ENSURE(tuple.Value().Maybe<TCoAtom>());
+ mode = tuple.Value().Cast<TCoAtom>();
} else if (name == "columns") {
YQL_ENSURE(tuple.Value().Maybe<TExprList>());
columns = tuple.Value().Cast<TExprList>();
} else if (name == "primarykey") {
- YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>());
- primaryKey = tuple.Value().Cast<TCoAtomList>();
+ YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>());
+ primaryKey = tuple.Value().Cast<TCoAtomList>();
} else if (name == "partitionby") {
YQL_ENSURE(tuple.Value().Maybe<TCoAtomList>());
partitionBy = tuple.Value().Cast<TCoAtomList>();
@@ -183,11 +183,11 @@ TWriteTableSettings ParseWriteTableSettings(TExprList node, TExprContext& ctx) {
YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>());
orderBy = tuple.Value().Cast<TCoNameValueTupleList>();
} else if (name == "filter") {
- YQL_ENSURE(tuple.Value().Maybe<TCoLambda>());
- filter = tuple.Value().Cast<TCoLambda>();
+ YQL_ENSURE(tuple.Value().Maybe<TCoLambda>());
+ filter = tuple.Value().Cast<TCoLambda>();
} else if (name == "update") {
- YQL_ENSURE(tuple.Value().Maybe<TCoLambda>());
- update = tuple.Value().Cast<TCoLambda>();
+ YQL_ENSURE(tuple.Value().Maybe<TCoLambda>());
+ update = tuple.Value().Cast<TCoLambda>();
} else if (name == "index") {
YQL_ENSURE(tuple.Value().Maybe<TCoNameValueTupleList>());
auto index = Build<TCoIndex>(ctx, node.Pos());
@@ -315,31 +315,31 @@ TWriteRoleSettings ParseWriteRoleSettings(TExprList node, TExprContext& ctx) {
return ret;
}
-TCommitSettings ParseCommitSettings(NNodes::TCoCommit node, TExprContext& ctx) {
+TCommitSettings ParseCommitSettings(NNodes::TCoCommit node, TExprContext& ctx) {
if (!node.Settings()) {
- return TCommitSettings(Build<TCoNameValueTupleList>(ctx, node.Pos()).Done());
+ return TCommitSettings(Build<TCoNameValueTupleList>(ctx, node.Pos()).Done());
}
- TMaybeNode<TCoAtom> mode;
- TMaybeNode<TCoAtom> epoch;
+ TMaybeNode<TCoAtom> mode;
+ TMaybeNode<TCoAtom> epoch;
TVector<TExprBase> other;
if (node.Settings()) {
auto settings = node.Settings().Cast();
for (auto setting : settings) {
if (setting.Name() == "mode") {
- YQL_ENSURE(setting.Value().Maybe<TCoAtom>());
- mode = setting.Value().Cast<TCoAtom>();
+ YQL_ENSURE(setting.Value().Maybe<TCoAtom>());
+ mode = setting.Value().Cast<TCoAtom>();
} else if (setting.Name() == "epoch") {
- YQL_ENSURE(setting.Value().Maybe<TCoAtom>());
- epoch = setting.Value().Cast<TCoAtom>();
+ YQL_ENSURE(setting.Value().Maybe<TCoAtom>());
+ epoch = setting.Value().Cast<TCoAtom>();
} else {
other.push_back(setting);
}
}
}
- auto otherSettings = Build<TCoNameValueTupleList>(ctx, node.Pos())
+ auto otherSettings = Build<TCoNameValueTupleList>(ctx, node.Pos())
.Add(other)
.Done();
@@ -392,10 +392,10 @@ void TransformerStatsToYson(const TString& name, const IGraphTransformer::TStati
writer.OnKeyedItem("NewTypeNodes");
writer.OnInt64Scalar(stats.NewTypeNodes);
}
- if (stats.NewConstraintNodes > 0) {
- writer.OnKeyedItem("NewConstraintNodes");
- writer.OnInt64Scalar(stats.NewConstraintNodes);
- }
+ if (stats.NewConstraintNodes > 0) {
+ writer.OnKeyedItem("NewConstraintNodes");
+ writer.OnInt64Scalar(stats.NewConstraintNodes);
+ }
if (stats.Repeats > 0) {
writer.OnKeyedItem("Repeats");
writer.OnUint64Scalar(stats.Repeats);
@@ -419,99 +419,99 @@ void TransformerStatsToYson(const TString& name, const IGraphTransformer::TStati
}
bool FillUsedFilesImpl(
- const TExprNode& node,
- TUserDataTable& files,
- const TTypeAnnotationContext& types,
+ const TExprNode& node,
+ TUserDataTable& files,
+ const TTypeAnnotationContext& types,
TExprContext& ctx,
const TUserDataTable& crutches,
TNodeSet& visited)
-{
+{
if (!visited.insert(&node).second) {
return true;
}
- if (node.IsCallable("FilePath") || node.IsCallable("FileContent")) {
+ if (node.IsCallable("FilePath") || node.IsCallable("FileContent")) {
const auto& name = node.Head().Content();
const auto block = types.UserDataStorage->FindUserDataBlock(name);
- if (!block) {
+ if (!block) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "File not found: " << name));
- return false;
- }
- else {
+ return false;
+ }
+ else {
auto iter = files.insert({ TUserDataStorage::ComposeUserDataKey(name), *block }).first;
- iter->second.Usage.Set(node.IsCallable("FilePath") ? EUserDataBlockUsage::Path : EUserDataBlockUsage::Content);
- }
- }
-
- if (node.IsCallable("FolderPath")) {
+ iter->second.Usage.Set(node.IsCallable("FilePath") ? EUserDataBlockUsage::Path : EUserDataBlockUsage::Content);
+ }
+ }
+
+ if (node.IsCallable("FolderPath")) {
const auto& name = node.Head().Content();
auto blocks = types.UserDataStorage->FindUserDataFolder(name);
- if (!blocks) {
+ if (!blocks) {
ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "Folder not found: " << name));
- return false;
- } else {
- for (const auto& x : *blocks) {
+ return false;
+ } else {
+ for (const auto& x : *blocks) {
auto iter = files.insert({ x.first, *x.second }).first;
- iter->second.Usage.Set(EUserDataBlockUsage::Path);
- }
- }
- }
-
- if (node.IsCallable("Udf") || node.IsCallable("ScriptUdf")) {
+ iter->second.Usage.Set(EUserDataBlockUsage::Path);
+ }
+ }
+ }
+
+ if (node.IsCallable("Udf") || node.IsCallable("ScriptUdf")) {
TStringBuf moduleName = node.Head().Content();
- if (node.IsCallable("Udf")) {
- TStringBuf funcName;
+ if (node.IsCallable("Udf")) {
+ TStringBuf funcName;
YQL_ENSURE(SplitUdfName(node.Head().Content(), moduleName, funcName));
- }
-
+ }
+
auto scriptType = NKikimr::NMiniKQL::ScriptTypeFromStr(moduleName);
if (node.IsCallable("ScriptUdf")) {
moduleName = NKikimr::NMiniKQL::ScriptTypeAsStr(NKikimr::NMiniKQL::CanonizeScriptType(scriptType));
}
- bool addSysModule = true;
- TString fileAlias;
- if (node.IsCallable("Udf")) {
- fileAlias = node.Child(6)->Content();
- } else {
- auto iterator = types.UdfModules.find(moduleName);
- // we have external UdfModule (not in preinstalled udfs)
- if (iterator != types.UdfModules.end()) {
- fileAlias = iterator->second;
+ bool addSysModule = true;
+ TString fileAlias;
+ if (node.IsCallable("Udf")) {
+ fileAlias = node.Child(6)->Content();
+ } else {
+ auto iterator = types.UdfModules.find(moduleName);
+ // we have external UdfModule (not in preinstalled udfs)
+ if (iterator != types.UdfModules.end()) {
+ fileAlias = iterator->second;
}
- }
-
- if (!fileAlias.empty()) {
- addSysModule = false;
- const auto block = types.UserDataStorage->FindUserDataBlock(fileAlias);
- if (!block) {
- ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "File not found: " << fileAlias));
- return false;
- } else {
- files.emplace(TUserDataStorage::ComposeUserDataKey(fileAlias), *block).first->second.Usage.Set(EUserDataBlockUsage::Udf);
+ }
+
+ if (!fileAlias.empty()) {
+ addSysModule = false;
+ const auto block = types.UserDataStorage->FindUserDataBlock(fileAlias);
+ if (!block) {
+ ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), TStringBuilder() << "File not found: " << fileAlias));
+ return false;
+ } else {
+ files.emplace(TUserDataStorage::ComposeUserDataKey(fileAlias), *block).first->second.Usage.Set(EUserDataBlockUsage::Udf);
}
- }
+ }
if (moduleName == TStringBuf("Geo")) {
const auto geobase = TUserDataKey::File(TStringBuf("/home/geodata6.bin"));
- if (const auto block = types.UserDataStorage->FindUserDataBlock(geobase)) {
- files.emplace(geobase, *block).first->second.Usage.Set(EUserDataBlockUsage::Path);
- } else {
- const auto it = crutches.find(geobase);
- if (crutches.cend() != it) {
- auto pragma = it->second;
- types.UserDataStorage->TryFillUserDataToken(pragma);
- types.UserDataStorage->AddUserDataBlock(geobase, pragma);
- files.emplace(geobase, pragma).first->second.Usage.Set(EUserDataBlockUsage::Path);
- }
- }
- }
-
- if (addSysModule) {
+ if (const auto block = types.UserDataStorage->FindUserDataBlock(geobase)) {
+ files.emplace(geobase, *block).first->second.Usage.Set(EUserDataBlockUsage::Path);
+ } else {
+ const auto it = crutches.find(geobase);
+ if (crutches.cend() != it) {
+ auto pragma = it->second;
+ types.UserDataStorage->TryFillUserDataToken(pragma);
+ types.UserDataStorage->AddUserDataBlock(geobase, pragma);
+ files.emplace(geobase, pragma).first->second.Usage.Set(EUserDataBlockUsage::Path);
+ }
+ }
+ }
+
+ if (addSysModule) {
auto pathWithMd5 = types.UdfResolver->GetSystemModulePath(moduleName);
YQL_ENSURE(pathWithMd5);
- TUserDataBlock sysBlock;
- sysBlock.Type = EUserDataType::PATH;
+ TUserDataBlock sysBlock;
+ sysBlock.Type = EUserDataType::PATH;
sysBlock.Data = pathWithMd5->Path;
sysBlock.Usage.Set(EUserDataBlockUsage::Udf);
@@ -534,17 +534,17 @@ bool FillUsedFilesImpl(
types.UserDataStorage->AddUserDataBlock(key, sysBlock);
}
}
- }
- }
-
- bool childrenOk = true;
- for (auto& child : node.Children()) {
+ }
+ }
+
+ bool childrenOk = true;
+ for (auto& child : node.Children()) {
childrenOk = FillUsedFilesImpl(*child, files, types, ctx, crutches, visited) && childrenOk;
- }
-
- return childrenOk;
-}
-
+ }
+
+ return childrenOk;
+}
+
static void GetToken(const TString& string, TString& out, const TTypeAnnotationContext& type) {
auto separator = string.find(":");
const auto p0 = string.substr(0, separator);
@@ -626,50 +626,50 @@ bool FillUsedFiles(
std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> FreezeUsedFiles(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter, const TUserDataTable& crutches) {
if (!FillUsedFiles(node, files, types, ctx, crutches)) {
- return SyncError();
+ return SyncError();
}
auto future = FreezeUserDataTableIfNeeded(types.UserDataStorage, files, urlDownloadFilter);
- if (future.Wait(TDuration::Zero())) {
- files = future.GetValue()();
- return SyncOk();
- }
- else {
- return std::make_pair(IGraphTransformer::TStatus::Async, future.Apply(
- [](const NThreading::TFuture<std::function<TUserDataTable()>>& completedFuture) {
- return TAsyncTransformCallback([completedFuture](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- output = input;
- try {
- completedFuture.GetValue()();
- }
- catch (const std::exception& e) {
+ if (future.Wait(TDuration::Zero())) {
+ files = future.GetValue()();
+ return SyncOk();
+ }
+ else {
+ return std::make_pair(IGraphTransformer::TStatus::Async, future.Apply(
+ [](const NThreading::TFuture<std::function<TUserDataTable()>>& completedFuture) {
+ return TAsyncTransformCallback([completedFuture](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ output = input;
+ try {
+ completedFuture.GetValue()();
+ }
+ catch (const std::exception& e) {
auto inputPos = ctx.GetPosition(input->Pos());
- TIssueScopeGuard issueScope(ctx.IssueManager, [&]() {
+ TIssueScopeGuard issueScope(ctx.IssueManager, [&]() {
return MakeIntrusive<TIssue>(YqlIssue(inputPos, TIssuesIds::UNEXPECTED));
- });
+ });
ctx.AddError(ExceptionToIssue(e, inputPos));
input->SetState(TExprNode::EState::Error);
- return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error);
- }
- catch (...) {
+ return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error);
+ }
+ catch (...) {
auto inputPos = ctx.GetPosition(input->Pos());
- TIssueScopeGuard issueScope(ctx.IssueManager, [&]() {
+ TIssueScopeGuard issueScope(ctx.IssueManager, [&]() {
return MakeIntrusive<TIssue>(YqlIssue(inputPos, TIssuesIds::UNEXPECTED));
- });
+ });
ctx.AddError(YqlIssue(inputPos, TIssuesIds::UNEXPECTED, CurrentExceptionMessage()));
input->SetState(TExprNode::EState::Error);
- return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error);
- }
-
+ return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error);
+ }
+
input->SetState(TExprNode::EState::ExecutionRequired);
- return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat);
- });
- }));
- }
+ return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Repeat);
+ });
+ }));
+ }
}
bool FreezeUsedFilesSync(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter) {
- if (!FillUsedFiles(node, files, types, ctx)) {
+ if (!FillUsedFiles(node, files, types, ctx)) {
return false;
}
@@ -728,36 +728,36 @@ void WriteStream(NYson::TYsonWriter& writer, const TExprNode* node, const TExprN
return;
}
- if (!node->GetTypeAnn()) {
- return;
- }
-
- TVector<const TExprNode*> applyStreamChildren;
- if (TCoApply::Match(node)) {
- switch (node->GetTypeAnn()->GetKind()) {
- case ETypeAnnotationKind::Stream:
+ if (!node->GetTypeAnn()) {
+ return;
+ }
+
+ TVector<const TExprNode*> applyStreamChildren;
+ if (TCoApply::Match(node)) {
+ switch (node->GetTypeAnn()->GetKind()) {
+ case ETypeAnnotationKind::Stream:
case ETypeAnnotationKind::Flow:
- case ETypeAnnotationKind::List:
- break;
- default:
- return;
- }
-
- for (size_t i = 1; i < node->ChildrenSize(); ++i) {
+ case ETypeAnnotationKind::List:
+ break;
+ default:
+ return;
+ }
+
+ for (size_t i = 1; i < node->ChildrenSize(); ++i) {
if (IsFlowOrStream(node->Child(i))) {
- applyStreamChildren.push_back(node->Child(i));
- } else if (node->Child(i)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) {
- if (node->Child(i)->IsCallable("ForwardList")) {
- applyStreamChildren.push_back(node->Child(i)->Child(0));
+ applyStreamChildren.push_back(node->Child(i));
+ } else if (node->Child(i)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) {
+ if (node->Child(i)->IsCallable("ForwardList")) {
+ applyStreamChildren.push_back(node->Child(i)->Child(0));
} else if (node->Child(i)->IsCallable("Collect") && IsFlowOrStream(node->Child(i)->HeadPtr().Get())) {
- applyStreamChildren.push_back(node->Child(i)->Child(0));
- }
- }
- }
- if (applyStreamChildren.size() == 1) {
- WriteStream(writer, applyStreamChildren.front(), source);
- }
- }
+ applyStreamChildren.push_back(node->Child(i)->Child(0));
+ }
+ }
+ }
+ if (applyStreamChildren.size() == 1) {
+ WriteStream(writer, applyStreamChildren.front(), source);
+ }
+ }
else if (!TCoExtendBase::Match(node) && node->ChildrenSize() > 0) {
WriteStream(writer, node->Child(0), source);
}
@@ -786,10 +786,10 @@ void WriteStream(NYson::TYsonWriter& writer, const TExprNode* node, const TExprN
writer.OnEndList();
}
- if (TCoSwitch::Match(node)) {
+ if (TCoSwitch::Match(node)) {
writer.OnKeyedItem("Children");
writer.OnBeginList();
- for (size_t i = 3; i < node->ChildrenSize(); i += 2) {
+ for (size_t i = 3; i < node->ChildrenSize(); i += 2) {
writer.OnListItem();
writer.OnBeginList();
WriteStream(writer, node->Child(i)->Child(1), node->Child(i)->Head().Child(0));
@@ -802,7 +802,7 @@ void WriteStream(NYson::TYsonWriter& writer, const TExprNode* node, const TExprN
if (TCoExtendBase::Match(node) && node->ChildrenSize() > 0) {
writer.OnKeyedItem("Children");
writer.OnBeginList();
- for (size_t i = 0; i < node->ChildrenSize(); ++i) {
+ for (size_t i = 0; i < node->ChildrenSize(); ++i) {
writer.OnListItem();
writer.OnBeginList();
WriteStream(writer, node->Child(i), source);
@@ -812,18 +812,18 @@ void WriteStream(NYson::TYsonWriter& writer, const TExprNode* node, const TExprN
writer.OnEndList();
}
- if (TCoApply::Match(node) && applyStreamChildren.size() > 1) {
- writer.OnKeyedItem("Children");
- writer.OnBeginList();
- for (auto child: applyStreamChildren) {
- writer.OnListItem();
- writer.OnBeginList();
- WriteStream(writer, child, source);
- writer.OnEndList();
- }
- writer.OnEndList();
- }
-
+ if (TCoApply::Match(node) && applyStreamChildren.size() > 1) {
+ writer.OnKeyedItem("Children");
+ writer.OnBeginList();
+ for (auto child: applyStreamChildren) {
+ writer.OnListItem();
+ writer.OnBeginList();
+ WriteStream(writer, child, source);
+ writer.OnEndList();
+ }
+ writer.OnEndList();
+ }
+
writer.OnEndMap();
}
@@ -835,45 +835,45 @@ void WriteStreams(NYson::TYsonWriter& writer, TStringBuf name, const NNodes::TCo
}
double GetDataReplicationFactor(double factor, const TExprNode* node, const TExprNode* stream, TExprContext& ctx) {
- if (node == stream) {
- return factor;
- }
-
+ if (node == stream) {
+ return factor;
+ }
+
if (!node->IsCallable()) {
- return factor;
- }
-
- if (TCoApply::Match(node)) {
- switch (node->GetTypeAnn()->GetKind()) {
- case ETypeAnnotationKind::Stream:
+ return factor;
+ }
+
+ if (TCoApply::Match(node)) {
+ switch (node->GetTypeAnn()->GetKind()) {
+ case ETypeAnnotationKind::Stream:
case ETypeAnnotationKind::Flow:
- case ETypeAnnotationKind::List: {
- double applyFactor = 0.0;
- for (size_t i = 1; i < node->ChildrenSize(); ++i) {
+ case ETypeAnnotationKind::List: {
+ double applyFactor = 0.0;
+ for (size_t i = 1; i < node->ChildrenSize(); ++i) {
if (IsFlowOrStream(node->Child(i))) {
applyFactor += GetDataReplicationFactor(factor, node->Child(i), stream, ctx);
- } else if (node->Child(i)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) {
- if (node->Child(i)->IsCallable("ForwardList")) {
+ } else if (node->Child(i)->GetTypeAnn()->GetKind() == ETypeAnnotationKind::List) {
+ if (node->Child(i)->IsCallable("ForwardList")) {
applyFactor += GetDataReplicationFactor(factor, node->Child(i)->Child(0), stream, ctx);
} else if (node->Child(i)->IsCallable("Collect") && IsFlowOrStream(node->Child(i)->HeadPtr().Get())) {
applyFactor += GetDataReplicationFactor(factor, node->Child(i)->Child(0), stream, ctx);
- }
- }
- }
- factor = 2.0 * Max(1.0, applyFactor);
- break;
- }
- default:
- break;
- }
- return factor;
- }
-
+ }
+ }
+ }
+ factor = 2.0 * Max(1.0, applyFactor);
+ break;
+ }
+ default:
+ break;
+ }
+ return factor;
+ }
+
if (!TCoExtendBase::Match(node) && node->ChildrenSize() > 0) {
factor = GetDataReplicationFactor(factor, node->Child(0), stream, ctx);
- }
-
- if (TCoFlatMapBase::Match(node)) {
+ }
+
+ if (TCoFlatMapBase::Match(node)) {
// TODO: check MapJoinCore input unique using constraints
if (const auto& lambda = node->Tail(); node->Head().IsCallable("SqueezeToDict") && lambda.Tail().IsCallable("MapJoinCore") && lambda.Tail().Child(1U) == &lambda.Head().Head()) {
TMaybe<bool> isMany;
@@ -896,49 +896,49 @@ double GetDataReplicationFactor(double factor, const TExprNode* node, const TExp
default:
break;
}
- }
- }
- else if (node->IsCallable("CommonJoinCore")) {
- factor *= 5.0;
- }
- else if (node->IsCallable("MapJoinCore")) {
- // TODO: check MapJoinCore input unique using constraints
- if (node->Child(1)->IsCallable("ToDict")) {
- TMaybe<bool> isMany;
- TMaybe<bool> isHashed;
- bool isCompact = false;
- TMaybe<ui64> itemsCount;
+ }
+ }
+ else if (node->IsCallable("CommonJoinCore")) {
+ factor *= 5.0;
+ }
+ else if (node->IsCallable("MapJoinCore")) {
+ // TODO: check MapJoinCore input unique using constraints
+ if (node->Child(1)->IsCallable("ToDict")) {
+ TMaybe<bool> isMany;
+ TMaybe<bool> isHashed;
+ bool isCompact = false;
+ TMaybe<ui64> itemsCount;
ParseToDictSettings(*node->Child(1), ctx, isMany, isHashed, itemsCount, isCompact);
- if (isMany.GetOrElse(true)) {
- factor *= 5.0;
- }
- }
- }
- else if (TCoSwitch::Match(node)) {
- double switchFactor = 0.0;
- for (size_t i = 3; i < node->ChildrenSize(); i += 2) {
+ if (isMany.GetOrElse(true)) {
+ factor *= 5.0;
+ }
+ }
+ }
+ else if (TCoSwitch::Match(node)) {
+ double switchFactor = 0.0;
+ for (size_t i = 3; i < node->ChildrenSize(); i += 2) {
switchFactor += GetDataReplicationFactor(factor, node->Child(i)->Child(1), node->Child(i)->Head().Child(0), ctx);
- }
- factor = Max(1.0, switchFactor);
- }
+ }
+ factor = Max(1.0, switchFactor);
+ }
else if (TCoExtendBase::Match(node) && node->ChildrenSize() > 0) {
- double extendFactor = 0.0;
- for (size_t i = 0; i < node->ChildrenSize(); ++i) {
+ double extendFactor = 0.0;
+ for (size_t i = 0; i < node->ChildrenSize(); ++i) {
extendFactor += GetDataReplicationFactor(factor, node->Child(i), stream, ctx);
- }
- factor = Max(1.0, extendFactor);
- }
+ }
+ factor = Max(1.0, extendFactor);
+ }
else if (TCoChopper::Match(node) || node->IsCallable("WideChopper")) {
- factor = GetDataReplicationFactor(factor, &node->Child(TCoChopper::idx_Handler)->Tail(), &node->Child(TCoChopper::idx_Handler)->Head().Tail(), ctx);
- }
-
- return factor;
-}
-
+ factor = GetDataReplicationFactor(factor, &node->Child(TCoChopper::idx_Handler)->Tail(), &node->Child(TCoChopper::idx_Handler)->Head().Tail(), ctx);
+ }
+
+ return factor;
+}
+
double GetDataReplicationFactor(const TExprNode& lambda, TExprContext& ctx) {
return GetDataReplicationFactor(1.0, lambda.Child(1), lambda.Head().ChildrenSize() > 0 ? lambda.Head().Child(0) : nullptr, ctx);
-}
-
+}
+
void WriteStatistics(NYson::TYsonWriter& writer, bool totalOnly, const THashMap<ui32, TOperationStatistics>& statistics) {
if (statistics.empty()) {
return;
diff --git a/ydb/library/yql/providers/common/provider/yql_provider.h b/ydb/library/yql/providers/common/provider/yql_provider.h
index 9e80cfd135..8cef2f470f 100644
--- a/ydb/library/yql/providers/common/provider/yql_provider.h
+++ b/ydb/library/yql/providers/common/provider/yql_provider.h
@@ -8,13 +8,13 @@
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <library/cpp/yson/writer.h>
-
-#include <util/generic/hash_set.h>
+
+#include <util/generic/hash_set.h>
#include <util/generic/string.h>
-#include <util/generic/strbuf.h>
-
-#include <utility>
-
+#include <util/generic/strbuf.h>
+
+#include <utility>
+
namespace NYson {
class TYsonWriter;
}
@@ -33,16 +33,16 @@ struct TOperationStatistics;
namespace NCommon {
struct TWriteTableSettings {
- NNodes::TMaybeNode<NNodes::TCoAtom> Mode;
+ NNodes::TMaybeNode<NNodes::TCoAtom> Mode;
NNodes::TMaybeNode<NNodes::TExprList> Columns;
- NNodes::TMaybeNode<NNodes::TCoAtomList> PrimaryKey;
+ NNodes::TMaybeNode<NNodes::TCoAtomList> PrimaryKey;
NNodes::TMaybeNode<NNodes::TCoAtomList> PartitionBy;
NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> OrderBy;
- NNodes::TMaybeNode<NNodes::TCoLambda> Filter;
- NNodes::TMaybeNode<NNodes::TCoLambda> Update;
+ NNodes::TMaybeNode<NNodes::TCoLambda> Filter;
+ NNodes::TMaybeNode<NNodes::TCoLambda> Update;
NNodes::TMaybeNode<NNodes::TCoIndexList> Indexes;
NNodes::TMaybeNode<NNodes::TCoChangefeedList> Changefeeds;
- NNodes::TCoNameValueTupleList Other;
+ NNodes::TCoNameValueTupleList Other;
NNodes::TMaybeNode<NNodes::TExprList> ColumnFamilies;
NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> TableSettings;
NNodes::TMaybeNode<NNodes::TCoNameValueTupleList> AlterActions;
@@ -63,14 +63,14 @@ struct TWriteRoleSettings {
struct TCommitSettings
{
TPositionHandle Pos;
- NNodes::TMaybeNode<NNodes::TCoAtom> Mode;
- NNodes::TMaybeNode<NNodes::TCoAtom> Epoch;
- NNodes::TCoNameValueTupleList Other;
+ NNodes::TMaybeNode<NNodes::TCoAtom> Mode;
+ NNodes::TMaybeNode<NNodes::TCoAtom> Epoch;
+ NNodes::TCoNameValueTupleList Other;
- TCommitSettings(NNodes::TCoNameValueTupleList other)
+ TCommitSettings(NNodes::TCoNameValueTupleList other)
: Other(other) {}
- NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx) const;
+ NNodes::TCoNameValueTupleList BuildNode(TExprContext& ctx) const;
bool EnsureModeEmpty(TExprContext& ctx);
bool EnsureEpochEmpty(TExprContext& ctx);
@@ -81,7 +81,7 @@ const TStructExprType* BuildCommonTableListType(TExprContext& ctx);
TExprNode::TPtr BuildTypeExpr(TPositionHandle pos, const TTypeAnnotationNode& ann, TExprContext& ctx);
-bool HasResOrPullOption(const TExprNode& node, const TStringBuf& option);
+bool HasResOrPullOption(const TExprNode& node, const TStringBuf& option);
TVector<TString> GetResOrPullColumnHints(const TExprNode& node);
@@ -89,7 +89,7 @@ TWriteTableSettings ParseWriteTableSettings(NNodes::TExprList node, TExprContext
TWriteRoleSettings ParseWriteRoleSettings(NNodes::TExprList node, TExprContext& ctx);
-TCommitSettings ParseCommitSettings(NNodes::TCoCommit node, TExprContext& ctx);
+TCommitSettings ParseCommitSettings(NNodes::TCoCommit node, TExprContext& ctx);
TString FullTableName(const TStringBuf& cluster, const TStringBuf& table);
@@ -103,11 +103,11 @@ void TransformerStatsToYson(const TString& name, const IGraphTransformer::TStati
void FillSecureParams(const TExprNode::TPtr& node, const TTypeAnnotationContext& types, THashMap<TString, TString>& secureParams);
bool FillUsedFiles(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const TUserDataTable& crutches = {});
-
+
std::pair<IGraphTransformer::TStatus, TAsyncTransformCallbackFuture> FreezeUsedFiles(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter, const TUserDataTable& crutches = {});
bool FreezeUsedFilesSync(const TExprNode& node, TUserDataTable& files, const TTypeAnnotationContext& types, TExprContext& ctx, const std::function<bool(const TString&)>& urlDownloadFilter);
-
+
void WriteColumns(NYson::TYsonWriter& writer, const NNodes::TExprBase& columns);
TString SerializeExpr(TExprContext& ctx, const TExprNode& expr, bool withTypes = false);
@@ -117,7 +117,7 @@ void WriteStream(NYson::TYsonWriter& writer, const TExprNode* node, const TExprN
void WriteStreams(NYson::TYsonWriter& writer, TStringBuf name, const NNodes::TCoLambda& lambda);
double GetDataReplicationFactor(const TExprNode& lambda, TExprContext& ctx);
-
+
void WriteStatistics(NYson::TYsonWriter& writer, bool totalOnly, const THashMap<ui32, TOperationStatistics>& statistics);
} // namespace NCommon
diff --git a/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp b/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp
index 93b84f6db0..34caa36259 100644
--- a/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp
+++ b/ydb/library/yql/providers/common/provider/yql_table_lookup.cpp
@@ -13,7 +13,7 @@ namespace {
template<typename TCompare>
TExprBase BuildColumnCompare(TExprBase row, const TString& columnName, TExprBase value, TExprContext& ctx) {
auto compare = Build<TCompare>(ctx, row.Pos())
- .template Left<TCoMember>()
+ .template Left<TCoMember>()
.Struct(row)
.Name().Build(columnName)
.Build()
@@ -22,13 +22,13 @@ TExprBase BuildColumnCompare(TExprBase row, const TString& columnName, TExprBase
TMaybeNode<TExprBase> ret = compare;
- auto rowType = row.Ref().GetTypeAnn()->Cast<TStructExprType>();
+ auto rowType = row.Ref().GetTypeAnn()->Cast<TStructExprType>();
auto columnType = rowType->GetItems()[*rowType->FindItem(columnName)]->GetItemType();
if (columnType->GetKind() == ETypeAnnotationKind::Optional) {
- ret = Build<TCoCoalesce>(ctx, row.Pos())
+ ret = Build<TCoCoalesce>(ctx, row.Pos())
.Predicate(compare)
- .template Value<TCoBool>()
+ .template Value<TCoBool>()
.Literal().Build("false")
.Build()
.Done();
@@ -39,7 +39,7 @@ TExprBase BuildColumnCompare(TExprBase row, const TString& columnName, TExprBase
TMaybeNode<TExprBase> CombinePredicatesAnd(TMaybeNode<TExprBase> left, TMaybeNode<TExprBase> right, TExprContext& ctx) {
if (left && right) {
- return Build<TCoAnd>(ctx, left.Cast().Pos())
+ return Build<TCoAnd>(ctx, left.Cast().Pos())
.Add({left.Cast(), right.Cast()})
.Done();
} else if (left) {
@@ -53,7 +53,7 @@ TMaybeNode<TExprBase> CombinePredicatesAnd(TMaybeNode<TExprBase> left, TMaybeNod
TMaybeNode<TExprBase> CombinePredicatesOr(TMaybeNode<TExprBase> left, TMaybeNode<TExprBase> right, TExprContext& ctx) {
if (left && right) {
- return Build<TCoOr>(ctx, left.Cast().Pos())
+ return Build<TCoOr>(ctx, left.Cast().Pos())
.Add({left.Cast(), right.Cast()})
.Done();
}
@@ -111,7 +111,7 @@ TMaybeNode<TExprBase> BuildColumnRangePredicate(const TString& column, const TCo
}
if (fromPredicate && toPredicate) {
- return Build<TCoAnd>(ctx, row.Pos())
+ return Build<TCoAnd>(ctx, row.Pos())
.Add({fromPredicate.Cast(), toPredicate.Cast()})
.Done();
} else if (fromPredicate) {
@@ -645,23 +645,23 @@ TTableLookupBuilder CollectLookups(TExprBase row, TExprBase predicate,
return CombineLookupsOr(row, keyColumns, builders.data(), builders.size(), ctx);
}
- TMaybeNode<TCoCompare> maybeCompare = predicate.Maybe<TCoCompare>();
- if (auto maybeLiteral = predicate.Maybe<TCoCoalesce>().Value().Maybe<TCoBool>().Literal()) {
+ TMaybeNode<TCoCompare> maybeCompare = predicate.Maybe<TCoCompare>();
+ if (auto maybeLiteral = predicate.Maybe<TCoCoalesce>().Value().Maybe<TCoBool>().Literal()) {
if (maybeLiteral.Cast().Value() == "false") {
- maybeCompare = predicate.Cast<TCoCoalesce>().Predicate().Maybe<TCoCompare>();
+ maybeCompare = predicate.Cast<TCoCoalesce>().Predicate().Maybe<TCoCompare>();
}
}
TTableLookupBuilder fullScan(keyColumns, TKeyRangeBuilder(keyColumns, predicate));
auto getRowMember = [row] (TExprBase expr) {
- if (auto maybeMember = expr.Maybe<TCoMember>()) {
+ if (auto maybeMember = expr.Maybe<TCoMember>()) {
if (maybeMember.Cast().Struct().Raw() == row.Raw()) {
return maybeMember;
}
}
- return TMaybeNode<TCoMember>();
+ return TMaybeNode<TCoMember>();
};
auto getTableLookup = [&keyColumns, fullScan] (const TStringBuf& column, const TColumnRange& range,
@@ -677,7 +677,7 @@ TTableLookupBuilder CollectLookups(TExprBase row, TExprBase predicate,
};
auto getNullComparePredicate = [&ctx](TExprBase value, TExprBase rowValue) -> TMaybeNode<TExprBase> {
- if (ctx.AllowNullCompare || rowValue.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional) {
+ if (ctx.AllowNullCompare || rowValue.Ref().GetTypeAnn()->GetKind() != ETypeAnnotationKind::Optional) {
return TMaybeNode<TExprBase>();
}
@@ -698,10 +698,10 @@ TTableLookupBuilder CollectLookups(TExprBase row, TExprBase predicate,
bool reverseCompare = false;
if (maybeLeftMember) {
maybeMember = maybeLeftMember;
- maybeValue = ctx.GetValueFunc(right, maybeMember.Cast().Ref().GetTypeAnn(), ctx.ExprCtx);
+ maybeValue = ctx.GetValueFunc(right, maybeMember.Cast().Ref().GetTypeAnn(), ctx.ExprCtx);
} else if (maybeRightMember) {
maybeMember = maybeRightMember;
- maybeValue = ctx.GetValueFunc(left, maybeMember.Cast().Ref().GetTypeAnn(), ctx.ExprCtx);
+ maybeValue = ctx.GetValueFunc(left, maybeMember.Cast().Ref().GetTypeAnn(), ctx.ExprCtx);
reverseCompare = true;
}
@@ -714,29 +714,29 @@ TTableLookupBuilder CollectLookups(TExprBase row, TExprBase predicate,
TMaybe<TColumnRange> columnRange;
- if (maybeCompare.Maybe<TCoCmpEqual>()) {
+ if (maybeCompare.Maybe<TCoCmpEqual>()) {
columnRange = TColumnRange::MakePoint(value);
}
- if (maybeCompare.Maybe<TCoCmpLess>()) {
+ if (maybeCompare.Maybe<TCoCmpLess>()) {
columnRange = reverseCompare
? MakeColumnRange(TRangeBound::MakeExclusive(value), TRangeBound::MakeUndefined(), ctx)
: MakeColumnRange(TRangeBound::MakeUndefined(), TRangeBound::MakeExclusive(value), ctx);
}
- if (maybeCompare.Maybe<TCoCmpLessOrEqual>()) {
+ if (maybeCompare.Maybe<TCoCmpLessOrEqual>()) {
columnRange = reverseCompare
? MakeColumnRange(TRangeBound::MakeInclusive(value), TRangeBound::MakeUndefined(), ctx)
: MakeColumnRange(TRangeBound::MakeUndefined(), TRangeBound::MakeInclusive(value), ctx);
}
- if (maybeCompare.Maybe<TCoCmpGreater>()) {
+ if (maybeCompare.Maybe<TCoCmpGreater>()) {
columnRange = reverseCompare
? MakeColumnRange(TRangeBound::MakeUndefined(), TRangeBound::MakeExclusive(value), ctx)
: MakeColumnRange(TRangeBound::MakeExclusive(value), TRangeBound::MakeUndefined(), ctx);
}
- if (maybeCompare.Maybe<TCoCmpGreaterOrEqual>()) {
+ if (maybeCompare.Maybe<TCoCmpGreaterOrEqual>()) {
columnRange = reverseCompare
? MakeColumnRange(TRangeBound::MakeUndefined(), TRangeBound::MakeInclusive(value), ctx)
: MakeColumnRange(TRangeBound::MakeInclusive(value), TRangeBound::MakeUndefined(), ctx);
@@ -803,27 +803,27 @@ TTableLookupBuilder CollectLookups(TExprBase row, TExprBase predicate,
return TTableLookupBuilder(keyColumns, keyRanges);
}
- if (auto maybeBool = predicate.Maybe<TCoCoalesce>().Value().Maybe<TCoBool>().Literal()) {
+ if (auto maybeBool = predicate.Maybe<TCoCoalesce>().Value().Maybe<TCoBool>().Literal()) {
if (maybeBool.Cast().Value() == "false") {
- TExprBase value = predicate.Cast<TCoCoalesce>().Predicate();
- TMaybeNode<TCoMember> maybeMember = getRowMember(value);
+ TExprBase value = predicate.Cast<TCoCoalesce>().Predicate();
+ TMaybeNode<TCoMember> maybeMember = getRowMember(value);
if (maybeMember) {
auto column = maybeMember.Cast().Name().Value();
auto columnRange = TColumnRange::MakePoint(
- Build<TCoBool>(ctx.ExprCtx, row.Pos()).Literal().Build("true").Done());
+ Build<TCoBool>(ctx.ExprCtx, row.Pos()).Literal().Build("true").Done());
return getTableLookup(column, columnRange, {});
}
}
}
- if (auto maybeBool = predicate.Maybe<TCoNot>().Value().Maybe<TCoCoalesce>().Value().Maybe<TCoBool>().Literal()) {
+ if (auto maybeBool = predicate.Maybe<TCoNot>().Value().Maybe<TCoCoalesce>().Value().Maybe<TCoBool>().Literal()) {
if (maybeBool.Cast().Value() == "true") {
- TExprBase value = predicate.Cast<TCoNot>().Value().Cast<TCoCoalesce>().Predicate();
- TMaybeNode<TCoMember> maybeMember = getRowMember(value);
+ TExprBase value = predicate.Cast<TCoNot>().Value().Cast<TCoCoalesce>().Predicate();
+ TMaybeNode<TCoMember> maybeMember = getRowMember(value);
if (maybeMember) {
auto column = maybeMember.Cast().Name().Value();
auto columnRange = TColumnRange::MakePoint(
- Build<TCoBool>(ctx.ExprCtx, row.Pos()).Literal().Build("false").Done());
+ Build<TCoBool>(ctx.ExprCtx, row.Pos()).Literal().Build("false").Done());
return getTableLookup(column, columnRange, {});
}
}
diff --git a/ydb/library/yql/providers/common/schema/expr/ya.make b/ydb/library/yql/providers/common/schema/expr/ya.make
index 60e34bc73a..de2264e4cd 100644
--- a/ydb/library/yql/providers/common/schema/expr/ya.make
+++ b/ydb/library/yql/providers/common/schema/expr/ya.make
@@ -1,15 +1,15 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(
g:yql
g:yql_ydb_core
)
-
-SRCS(
- yql_expr_schema.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ yql_expr_schema.cpp
+)
+
+PEERDIR(
library/cpp/yson
library/cpp/yson/node
ydb/library/yql/ast
@@ -17,6 +17,6 @@ PEERDIR(
ydb/library/yql/public/udf
ydb/library/yql/utils
ydb/library/yql/providers/common/schema/parser
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp b/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp
index 0bbaf5c339..9fd60b38c9 100644
--- a/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp
+++ b/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.cpp
@@ -1,19 +1,19 @@
-#include "yql_expr_schema.h"
+#include "yql_expr_schema.h"
#include <ydb/library/yql/providers/common/schema/parser/yql_type_parser.h>
#include <ydb/library/yql/ast/yql_expr_types.h>
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/utils/yql_panic.h>
#include <ydb/library/yql/public/udf/udf_data_type.h>
-
+
#include <library/cpp/yson/node/node_io.h>
#include <library/cpp/yson/node/node_builder.h>
#include <library/cpp/yson/writer.h>
-
+
#include <util/generic/map.h>
-#include <util/stream/str.h>
-
+#include <util/stream/str.h>
+
namespace NYql {
namespace NCommon {
@@ -98,57 +98,57 @@ class TExprTypeSaver: public TSaver<TExprTypeSaver<TSaver>> {
}
};
- struct TCallableAdaptor {
- const TCallableExprType* Type;
-
- TCallableAdaptor(const TCallableExprType* type)
- : Type(type)
- {
- }
-
- size_t GetOptionalArgsCount() const {
- return Type->GetOptionalArgumentsCount();
- }
-
- TStringBuf GetPayload() const {
- return Type->GetPayload();
- }
-
- const TTypeAnnotationNode* GetReturnType() const {
- return Type->GetReturnType();
- }
-
- size_t GetArgumentsCount() const {
- return Type->GetArgumentsSize();
- }
-
- TStringBuf GetArgumentName(size_t i) const {
- return Type->GetArguments().at(i).Name;
- }
-
- ui64 GetArgumentFlags(size_t i) const {
- return Type->GetArguments().at(i).Flags;
- }
-
- const TTypeAnnotationNode* GetArgumentType(size_t i) const {
- return Type->GetArguments().at(i).Type;
- }
- };
-
- void SaveErrorType(const TErrorExprType& errorType) {
- TBase::SaveTypeHeader("ErrorType");
- auto err = errorType.GetError();
- TBase::Writer.OnListItem();
- TBase::Writer.OnInt64Scalar(err.Position.Row);
- TBase::Writer.OnListItem();
- TBase::Writer.OnInt64Scalar(err.Position.Column);
- TBase::Writer.OnListItem();
- TBase::Writer.OnStringScalar(err.Position.File);
- TBase::Writer.OnListItem();
- TBase::Writer.OnStringScalar(err.Message);
- TBase::Writer.OnEndList();
- }
-
+ struct TCallableAdaptor {
+ const TCallableExprType* Type;
+
+ TCallableAdaptor(const TCallableExprType* type)
+ : Type(type)
+ {
+ }
+
+ size_t GetOptionalArgsCount() const {
+ return Type->GetOptionalArgumentsCount();
+ }
+
+ TStringBuf GetPayload() const {
+ return Type->GetPayload();
+ }
+
+ const TTypeAnnotationNode* GetReturnType() const {
+ return Type->GetReturnType();
+ }
+
+ size_t GetArgumentsCount() const {
+ return Type->GetArgumentsSize();
+ }
+
+ TStringBuf GetArgumentName(size_t i) const {
+ return Type->GetArguments().at(i).Name;
+ }
+
+ ui64 GetArgumentFlags(size_t i) const {
+ return Type->GetArguments().at(i).Flags;
+ }
+
+ const TTypeAnnotationNode* GetArgumentType(size_t i) const {
+ return Type->GetArguments().at(i).Type;
+ }
+ };
+
+ void SaveErrorType(const TErrorExprType& errorType) {
+ TBase::SaveTypeHeader("ErrorType");
+ auto err = errorType.GetError();
+ TBase::Writer.OnListItem();
+ TBase::Writer.OnInt64Scalar(err.Position.Row);
+ TBase::Writer.OnListItem();
+ TBase::Writer.OnInt64Scalar(err.Position.Column);
+ TBase::Writer.OnListItem();
+ TBase::Writer.OnStringScalar(err.Position.File);
+ TBase::Writer.OnListItem();
+ TBase::Writer.OnStringScalar(err.Message);
+ TBase::Writer.OnEndList();
+ }
+
public:
TExprTypeSaver(typename TBase::TConsumer& consumer)
: TBase(consumer)
@@ -208,11 +208,11 @@ public:
TBase::SaveTaggedType(*type->Cast<TTaggedExprType>());
break;
case ETypeAnnotationKind::Error:
- SaveErrorType(*type->Cast<TErrorExprType>());
- break;
- case ETypeAnnotationKind::Callable:
- TBase::SaveCallableType(TCallableAdaptor(type->Cast<TCallableExprType>()));
+ SaveErrorType(*type->Cast<TErrorExprType>());
break;
+ case ETypeAnnotationKind::Callable:
+ TBase::SaveCallableType(TCallableAdaptor(type->Cast<TCallableExprType>()));
+ break;
case ETypeAnnotationKind::Variant:
TBase::SaveVariantType(*type->Cast<TVariantExprType>());
break;
@@ -258,54 +258,54 @@ TString WriteTypeToYson(const TTypeAnnotationNode* type, NYson::EYsonFormat form
}
struct TExprTypeLoader {
- typedef const TTypeAnnotationNode* TType;
+ typedef const TTypeAnnotationNode* TType;
TExprContext& Ctx;
- TPosition Pos;
+ TPosition Pos;
- TExprTypeLoader(TExprContext& ctx, const TPosition& pos = TPosition())
+ TExprTypeLoader(TExprContext& ctx, const TPosition& pos = TPosition())
: Ctx(ctx)
- , Pos(pos)
+ , Pos(pos)
{
}
- TMaybe<TType> LoadVoidType(ui32 /*level*/) {
+ TMaybe<TType> LoadVoidType(ui32 /*level*/) {
return Ctx.MakeType<TVoidExprType>();
}
- TMaybe<TType> LoadNullType(ui32 /*level*/) {
+ TMaybe<TType> LoadNullType(ui32 /*level*/) {
return Ctx.MakeType<TNullExprType>();
}
- TMaybe<TType> LoadUnitType(ui32 /*level*/) {
+ TMaybe<TType> LoadUnitType(ui32 /*level*/) {
return Ctx.MakeType<TUnitExprType>();
}
- TMaybe<TType> LoadGenericType(ui32 /*level*/) {
+ TMaybe<TType> LoadGenericType(ui32 /*level*/) {
return Ctx.MakeType<TGenericExprType>();
}
- TMaybe<TType> LoadEmptyListType(ui32 /*level*/) {
+ TMaybe<TType> LoadEmptyListType(ui32 /*level*/) {
return Ctx.MakeType<TEmptyListExprType>();
}
- TMaybe<TType> LoadEmptyDictType(ui32 /*level*/) {
+ TMaybe<TType> LoadEmptyDictType(ui32 /*level*/) {
return Ctx.MakeType<TEmptyDictExprType>();
}
- TMaybe<TType> LoadDataType(const TString& dataType, ui32 /*level*/) {
- return Ctx.MakeType<TDataExprType>(NYql::NUdf::GetDataSlot(dataType));
+ TMaybe<TType> LoadDataType(const TString& dataType, ui32 /*level*/) {
+ return Ctx.MakeType<TDataExprType>(NYql::NUdf::GetDataSlot(dataType));
}
- TMaybe<TType> LoadDataTypeParams(const TString& dataType, const TString& paramOne, const TString& paramTwo, ui32 /*level*/) {
- auto ret = Ctx.MakeType<TDataExprParamsType>(NYql::NUdf::GetDataSlot(dataType), paramOne, paramTwo);
+ TMaybe<TType> LoadDataTypeParams(const TString& dataType, const TString& paramOne, const TString& paramTwo, ui32 /*level*/) {
+ auto ret = Ctx.MakeType<TDataExprParamsType>(NYql::NUdf::GetDataSlot(dataType), paramOne, paramTwo);
YQL_ENSURE(ret->Validate(TPosition(), Ctx));
return ret;
}
- TMaybe<TType> LoadResourceType(const TString& tag, ui32 /*level*/) {
- return Ctx.MakeType<TResourceExprType>(tag);
- }
- TMaybe<TType> LoadTaggedType(TType baseType, const TString& tag, ui32 /*level*/) {
- auto ret = Ctx.MakeType<TTaggedExprType>(baseType, tag);
- YQL_ENSURE(ret->Validate(TPosition(), Ctx));
- return ret;
- }
- TMaybe<TType> LoadErrorType(ui32 row, ui32 column, const TString& file, const TString& msg, ui32 /*level*/) {
+ TMaybe<TType> LoadResourceType(const TString& tag, ui32 /*level*/) {
+ return Ctx.MakeType<TResourceExprType>(tag);
+ }
+ TMaybe<TType> LoadTaggedType(TType baseType, const TString& tag, ui32 /*level*/) {
+ auto ret = Ctx.MakeType<TTaggedExprType>(baseType, tag);
+ YQL_ENSURE(ret->Validate(TPosition(), Ctx));
+ return ret;
+ }
+ TMaybe<TType> LoadErrorType(ui32 row, ui32 column, const TString& file, const TString& msg, ui32 /*level*/) {
return Ctx.MakeType<TErrorExprType>(TIssue(TPosition(column, row, file), msg));
- }
- TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 /*level*/) {
+ }
+ TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 /*level*/) {
TVector<const TItemExprType*> items;
for (auto& member: members) {
items.push_back(Ctx.MakeType<TItemExprType>(member.first, member.second));
@@ -314,40 +314,40 @@ struct TExprTypeLoader {
YQL_ENSURE(ret->Validate(TPosition(), Ctx));
return ret;
}
- TMaybe<TType> LoadListType(TType itemType, ui32 /*level*/) {
+ TMaybe<TType> LoadListType(TType itemType, ui32 /*level*/) {
return Ctx.MakeType<TListExprType>(itemType);
}
- TMaybe<TType> LoadStreamType(TType itemType, ui32 /*level*/) {
+ TMaybe<TType> LoadStreamType(TType itemType, ui32 /*level*/) {
return Ctx.MakeType<TStreamExprType>(itemType);
}
- TMaybe<TType> LoadOptionalType(TType itemType, ui32 /*level*/) {
+ TMaybe<TType> LoadOptionalType(TType itemType, ui32 /*level*/) {
return Ctx.MakeType<TOptionalExprType>(itemType);
}
- TMaybe<TType> LoadTupleType(const TVector<TType>& elements, ui32 /*level*/) {
+ TMaybe<TType> LoadTupleType(const TVector<TType>& elements, ui32 /*level*/) {
auto ret = Ctx.MakeType<TTupleExprType>(elements);
YQL_ENSURE(ret->Validate(TPosition(), Ctx));
return ret;
}
- TMaybe<TType> LoadDictType(TType keyType, TType valType, ui32 /*level*/) {
+ TMaybe<TType> LoadDictType(TType keyType, TType valType, ui32 /*level*/) {
auto ret = Ctx.MakeType<TDictExprType>(keyType, valType);
YQL_ENSURE(ret->Validate(TPosition(), Ctx));
return ret;
}
- TMaybe<TType> LoadCallableType(TType returnType, const TVector<TType>& argTypes, const TVector<TString>& argNames,
- const TVector<ui64>& argFlags, size_t optionalCount, const TString& payload, ui32 /*level*/) {
- YQL_ENSURE(argTypes.size() == argNames.size() && argTypes.size() == argFlags.size());
+ TMaybe<TType> LoadCallableType(TType returnType, const TVector<TType>& argTypes, const TVector<TString>& argNames,
+ const TVector<ui64>& argFlags, size_t optionalCount, const TString& payload, ui32 /*level*/) {
+ YQL_ENSURE(argTypes.size() == argNames.size() && argTypes.size() == argFlags.size());
TVector<TCallableExprType::TArgumentInfo> args;
- for (size_t i = 0; i < argTypes.size(); ++i) {
- args.emplace_back();
- args.back().Type = argTypes[i];
- args.back().Name = Ctx.AppendString(argNames[i]);
- args.back().Flags = argFlags[i];
- }
+ for (size_t i = 0; i < argTypes.size(); ++i) {
+ args.emplace_back();
+ args.back().Type = argTypes[i];
+ args.back().Name = Ctx.AppendString(argNames[i]);
+ args.back().Flags = argFlags[i];
+ }
auto ret = Ctx.MakeType<TCallableExprType>(returnType, args, optionalCount, Ctx.AppendString(payload));
YQL_ENSURE(ret->Validate(TPosition(), Ctx));
return ret;
- }
- TMaybe<TType> LoadVariantType(TType underlyingType, ui32 /*level*/) {
+ }
+ TMaybe<TType> LoadVariantType(TType underlyingType, ui32 /*level*/) {
auto ret = Ctx.MakeType<TVariantExprType>(underlyingType);
YQL_ENSURE(ret->Validate(TPosition(), Ctx));
return ret;
@@ -357,7 +357,7 @@ struct TExprTypeLoader {
}
};
-const TTypeAnnotationNode* ParseTypeFromYson(const TStringBuf yson, TExprContext& ctx, const TPosition& pos) {
+const TTypeAnnotationNode* ParseTypeFromYson(const TStringBuf yson, TExprContext& ctx, const TPosition& pos) {
NYT::TNode node;
TStringStream err;
if (!ParseYson(node, yson, err)) {
@@ -365,7 +365,7 @@ const TTypeAnnotationNode* ParseTypeFromYson(const TStringBuf yson, TExprContext
return nullptr;
}
- return ParseTypeFromYson(node, ctx, pos);
+ return ParseTypeFromYson(node, ctx, pos);
}
const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const TStringBuf yson, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos) {
@@ -379,9 +379,9 @@ const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const TStringBuf yson, TV
return ParseOrderAwareTypeFromYson(node, topLevelColumns, ctx, pos);
}
-const TTypeAnnotationNode* ParseTypeFromYson(const NYT::TNode& node, TExprContext& ctx, const TPosition& pos) {
- TExprTypeLoader loader(ctx, pos);
- return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr);
+const TTypeAnnotationNode* ParseTypeFromYson(const NYT::TNode& node, TExprContext& ctx, const TPosition& pos) {
+ TExprTypeLoader loader(ctx, pos);
+ return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr);
}
struct TOrderAwareExprTypeLoader: public TExprTypeLoader {
diff --git a/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h b/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h
index 397063052f..a15f3600c0 100644
--- a/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h
+++ b/ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h
@@ -5,18 +5,18 @@
#include <library/cpp/yson/consumer.h>
#include <library/cpp/yson/node/node.h>
-#include <util/generic/maybe.h>
-#include <util/generic/string.h>
-#include <util/generic/vector.h>
-#include <util/generic/strbuf.h>
-
-#include <functional>
-
-namespace NYql {
-
-class TTypeAnnotationNode;
-class TStructExprType;
-struct TExprContext;
+#include <util/generic/maybe.h>
+#include <util/generic/string.h>
+#include <util/generic/vector.h>
+#include <util/generic/strbuf.h>
+
+#include <functional>
+
+namespace NYql {
+
+class TTypeAnnotationNode;
+class TStructExprType;
+struct TExprContext;
namespace NCommon {
@@ -31,9 +31,9 @@ void SaveStructTypeToYson(NYson::TYsonConsumerBase& writer, const TStructExprTyp
NYT::TNode TypeToYsonNode(const TTypeAnnotationNode* type);
TString WriteTypeToYson(const TTypeAnnotationNode* type, NYT::NYson::EYsonFormat format = NYT::NYson::EYsonFormat::Binary);
-const TTypeAnnotationNode* ParseTypeFromYson(const TStringBuf yson, TExprContext& ctx, const TPosition& pos = TPosition());
+const TTypeAnnotationNode* ParseTypeFromYson(const TStringBuf yson, TExprContext& ctx, const TPosition& pos = TPosition());
const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const TStringBuf yson, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos = TPosition());
-const TTypeAnnotationNode* ParseTypeFromYson(const NYT::TNode& node, TExprContext& ctx, const TPosition& pos = TPosition());
+const TTypeAnnotationNode* ParseTypeFromYson(const NYT::TNode& node, TExprContext& ctx, const TPosition& pos = TPosition());
const TTypeAnnotationNode* ParseOrderAwareTypeFromYson(const NYT::TNode& node, TVector<TString>& topLevelColumns, TExprContext& ctx, const TPosition& pos = TPosition());
void WriteResOrPullType(NYson::TYsonConsumerBase& writer, const TTypeAnnotationNode* type,
diff --git a/ydb/library/yql/providers/common/schema/mkql/ya.make b/ydb/library/yql/providers/common/schema/mkql/ya.make
index 2ddbbdb345..0c5eb1d529 100644
--- a/ydb/library/yql/providers/common/schema/mkql/ya.make
+++ b/ydb/library/yql/providers/common/schema/mkql/ya.make
@@ -1,15 +1,15 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(
g:yql
g:yql_ydb_core
)
-
-SRCS(
- yql_mkql_schema.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ yql_mkql_schema.cpp
+)
+
+PEERDIR(
library/cpp/yson
library/cpp/yson/node
ydb/library/yql/minikql
@@ -17,8 +17,8 @@ PEERDIR(
ydb/library/yql/utils
ydb/library/yql/providers/common/codec
ydb/library/yql/providers/common/schema/parser
-)
-
-YQL_LAST_ABI_VERSION()
-
-END()
+)
+
+YQL_LAST_ABI_VERSION()
+
+END()
diff --git a/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.cpp b/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.cpp
index 0a34105714..3a8a18cfae 100644
--- a/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.cpp
+++ b/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.cpp
@@ -1,4 +1,4 @@
-#include "yql_mkql_schema.h"
+#include "yql_mkql_schema.h"
#include <ydb/library/yql/providers/common/schema/parser/yql_type_parser.h>
#include <ydb/library/yql/providers/common/codec/yql_codec.h>
@@ -10,13 +10,13 @@
#include <ydb/library/yql/public/udf/udf_type_inspection.h>
#include <ydb/library/yql/public/udf/udf_data_type.h>
#include <ydb/library/yql/utils/yql_panic.h>
-
+
#include <library/cpp/yson/node/node_io.h>
#include <library/cpp/yson/node/node_builder.h>
#include <library/cpp/yson/writer.h>
-#include <util/stream/str.h>
-
+#include <util/stream/str.h>
+
namespace NYql {
namespace NCommon {
@@ -24,47 +24,47 @@ template <template<typename> class TSaver>
class TRuntimeTypeSaver: public TSaver<TRuntimeTypeSaver<TSaver>> {
typedef TSaver<TRuntimeTypeSaver> TBase;
- struct TCallableAdaptor {
- const NKikimr::NMiniKQL::TCallableType* Type;
- NKikimr::NMiniKQL::TTypeInfoHelper TypeHelper;
- NKikimr::NUdf::TCallableTypeInspector CallableInspector;
-
- TCallableAdaptor(const NKikimr::NMiniKQL::TCallableType* type)
- : Type(type)
- , TypeHelper()
- , CallableInspector(TypeHelper, Type)
- {
- }
-
- size_t GetOptionalArgsCount() const {
- return CallableInspector.GetOptionalArgsCount();
- }
-
- TStringBuf GetPayload() const {
- return CallableInspector.GetPayload();
- }
-
- NKikimr::NMiniKQL::TType* GetReturnType() const {
- return Type->GetReturnType();
- }
-
- size_t GetArgumentsCount() const {
- return Type->GetArgumentsCount();
- }
-
- TStringBuf GetArgumentName(size_t i) const {
- return CallableInspector.GetArgumentName(i);
- }
-
- ui64 GetArgumentFlags(size_t i) const {
- return CallableInspector.GetArgumentFlags(i);
- }
-
- NKikimr::NMiniKQL::TType* GetArgumentType(size_t i) const {
- return Type->GetArgumentType(i);
- }
- };
-
+ struct TCallableAdaptor {
+ const NKikimr::NMiniKQL::TCallableType* Type;
+ NKikimr::NMiniKQL::TTypeInfoHelper TypeHelper;
+ NKikimr::NUdf::TCallableTypeInspector CallableInspector;
+
+ TCallableAdaptor(const NKikimr::NMiniKQL::TCallableType* type)
+ : Type(type)
+ , TypeHelper()
+ , CallableInspector(TypeHelper, Type)
+ {
+ }
+
+ size_t GetOptionalArgsCount() const {
+ return CallableInspector.GetOptionalArgsCount();
+ }
+
+ TStringBuf GetPayload() const {
+ return CallableInspector.GetPayload();
+ }
+
+ NKikimr::NMiniKQL::TType* GetReturnType() const {
+ return Type->GetReturnType();
+ }
+
+ size_t GetArgumentsCount() const {
+ return Type->GetArgumentsCount();
+ }
+
+ TStringBuf GetArgumentName(size_t i) const {
+ return CallableInspector.GetArgumentName(i);
+ }
+
+ ui64 GetArgumentFlags(size_t i) const {
+ return CallableInspector.GetArgumentFlags(i);
+ }
+
+ NKikimr::NMiniKQL::TType* GetArgumentType(size_t i) const {
+ return Type->GetArgumentType(i);
+ }
+ };
+
public:
TRuntimeTypeSaver(typename TBase::TConsumer& consumer)
: TBase(consumer)
@@ -120,7 +120,7 @@ public:
TBase::SaveDictType(*static_cast<const TDictType*>(type));
break;
case TType::EKind::Callable:
- TBase::SaveCallableType(TCallableAdaptor(static_cast<const TCallableType*>(type)));
+ TBase::SaveCallableType(TCallableAdaptor(static_cast<const TCallableType*>(type)));
break;
case TType::EKind::Tuple:
TBase::SaveTupleType(*static_cast<const TTupleType*>(type));
@@ -163,7 +163,7 @@ TString WriteTypeToYson(const NKikimr::NMiniKQL::TType* type, NYson::EYsonFormat
}
struct TRuntimeTypeLoader {
- typedef NKikimr::NMiniKQL::TType* TType;
+ typedef NKikimr::NMiniKQL::TType* TType;
NKikimr::NMiniKQL::TProgramBuilder& Builder;
IOutputStream& Err;
@@ -173,118 +173,118 @@ struct TRuntimeTypeLoader {
, Err(err)
{
}
- TMaybe<TType> LoadVoidType(ui32 /*level*/) {
+ TMaybe<TType> LoadVoidType(ui32 /*level*/) {
return Builder.NewVoid().GetStaticType();
}
- TMaybe<TType> LoadNullType(ui32 /*level*/) {
+ TMaybe<TType> LoadNullType(ui32 /*level*/) {
return Builder.NewNull().GetStaticType();
}
- TMaybe<TType> LoadUnitType(ui32 /*level*/) {
+ TMaybe<TType> LoadUnitType(ui32 /*level*/) {
return Builder.NewVoid().GetStaticType();
}
- TMaybe<TType> LoadGenericType(ui32 /*level*/) {
+ TMaybe<TType> LoadGenericType(ui32 /*level*/) {
return Builder.GetTypeEnvironment().GetTypeOfType();
}
- TMaybe<TType> LoadEmptyListType(ui32 /*level*/) {
+ TMaybe<TType> LoadEmptyListType(ui32 /*level*/) {
if (NKikimr::NMiniKQL::RuntimeVersion < 11) {
return Builder.NewListType(Builder.NewVoid().GetStaticType());
}
return Builder.GetTypeEnvironment().GetTypeOfEmptyList();
}
- TMaybe<TType> LoadEmptyDictType(ui32 /*level*/) {
+ TMaybe<TType> LoadEmptyDictType(ui32 /*level*/) {
if (NKikimr::NMiniKQL::RuntimeVersion < 11) {
return Builder.NewDictType(Builder.NewVoid().GetStaticType(), Builder.NewVoid().GetStaticType(), false);
}
return Builder.GetTypeEnvironment().GetTypeOfEmptyDict();
}
- TMaybe<TType> LoadDataType(const TString& dataType, ui32 /*level*/) {
- const auto slot = NUdf::FindDataSlot(dataType);
+ TMaybe<TType> LoadDataType(const TString& dataType, ui32 /*level*/) {
+ const auto slot = NUdf::FindDataSlot(dataType);
if (!slot) {
Err << "Unsupported data type: " << dataType;
- return Nothing();
+ return Nothing();
}
if (NKikimr::NUdf::EDataSlot::Decimal == slot) {
Err << "Decimal type without parameters.";
- return Nothing();
+ return Nothing();
}
- return Builder.NewDataType(NUdf::GetDataTypeInfo(*slot).TypeId);
+ return Builder.NewDataType(NUdf::GetDataTypeInfo(*slot).TypeId);
}
- TMaybe<TType> LoadDataTypeParams(const TString& dataType, const TString& paramOne, const TString& paramTwo, ui32 /*level*/) {
- const auto slot = NUdf::FindDataSlot(dataType);
+ TMaybe<TType> LoadDataTypeParams(const TString& dataType, const TString& paramOne, const TString& paramTwo, ui32 /*level*/) {
+ const auto slot = NUdf::FindDataSlot(dataType);
if (!slot) {
Err << "Unsupported data type: " << dataType;
- return Nothing();
+ return Nothing();
}
if (NKikimr::NUdf::EDataSlot::Decimal != slot) {
Err << "Unexpected parameters for type: " << dataType;
- return Nothing();
+ return Nothing();
}
return Builder.NewDecimalType(FromString<ui8>(paramOne), FromString<ui8>(paramTwo));
}
- TMaybe<TType> LoadResourceType(const TString& tag, ui32 /*level*/) {
- return Builder.NewResourceType(tag);
- }
- TMaybe<TType> LoadTaggedType(TType baseType, const TString& /*tag*/, ui32 /*level*/) {
+ TMaybe<TType> LoadResourceType(const TString& tag, ui32 /*level*/) {
+ return Builder.NewResourceType(tag);
+ }
+ TMaybe<TType> LoadTaggedType(TType baseType, const TString& /*tag*/, ui32 /*level*/) {
return baseType;
- }
- TMaybe<TType> LoadErrorType(ui32 /*row*/, ui32 /*column*/, const TString& /*file*/, const TString& msg, ui32 /*level*/) {
- Err << msg;
- return Nothing();
- }
- TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 /*level*/) {
+ }
+ TMaybe<TType> LoadErrorType(ui32 /*row*/, ui32 /*column*/, const TString& /*file*/, const TString& msg, ui32 /*level*/) {
+ Err << msg;
+ return Nothing();
+ }
+ TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 /*level*/) {
auto structType = Builder.NewEmptyStructType();
for (auto& member : members) {
structType = Builder.NewStructType(structType, member.first, member.second);
}
return structType;
}
- TMaybe<TType> LoadListType(TType itemType, ui32 /*level*/) {
+ TMaybe<TType> LoadListType(TType itemType, ui32 /*level*/) {
return Builder.NewListType(itemType);
}
- TMaybe<TType> LoadStreamType(TType itemType, ui32 /*level*/) {
+ TMaybe<TType> LoadStreamType(TType itemType, ui32 /*level*/) {
return Builder.NewStreamType(itemType);
}
- TMaybe<TType> LoadOptionalType(TType itemType, ui32 /*level*/) {
+ TMaybe<TType> LoadOptionalType(TType itemType, ui32 /*level*/) {
return Builder.NewOptionalType(itemType);
}
- TMaybe<TType> LoadTupleType(const TVector<TType>& elements, ui32 /*level*/) {
+ TMaybe<TType> LoadTupleType(const TVector<TType>& elements, ui32 /*level*/) {
return Builder.NewTupleType(elements);
}
- TMaybe<TType> LoadDictType(TType keyType, TType valType, ui32 /*level*/) {
+ TMaybe<TType> LoadDictType(TType keyType, TType valType, ui32 /*level*/) {
return Builder.NewDictType(keyType, valType, false);
}
- TMaybe<TType> LoadCallableType(TType returnType, const TVector<TType>& argTypes, const TVector<TString>& argNames,
- const TVector<ui64>& argFlags, size_t optionalCount, const TString& payload, ui32 /*level*/) {
-
- YQL_ENSURE(argTypes.size() == argNames.size() && argTypes.size() == argFlags.size());
-
- NKikimr::NMiniKQL::TCallableTypeBuilder callableTypeBuilder(Builder.GetTypeEnvironment(), "", returnType);
- for (size_t i = 0; i < argTypes.size(); ++i) {
- callableTypeBuilder.Add(argTypes[i]);
+ TMaybe<TType> LoadCallableType(TType returnType, const TVector<TType>& argTypes, const TVector<TString>& argNames,
+ const TVector<ui64>& argFlags, size_t optionalCount, const TString& payload, ui32 /*level*/) {
+
+ YQL_ENSURE(argTypes.size() == argNames.size() && argTypes.size() == argFlags.size());
+
+ NKikimr::NMiniKQL::TCallableTypeBuilder callableTypeBuilder(Builder.GetTypeEnvironment(), "", returnType);
+ for (size_t i = 0; i < argTypes.size(); ++i) {
+ callableTypeBuilder.Add(argTypes[i]);
if (!argNames[i].empty()) {
- callableTypeBuilder.SetArgumentName(argNames[i]);
- }
-
- if (argFlags[i] != 0) {
- callableTypeBuilder.SetArgumentFlags(argFlags[i]);
- }
- }
-
- callableTypeBuilder.SetOptionalArgs(optionalCount);
+ callableTypeBuilder.SetArgumentName(argNames[i]);
+ }
+
+ if (argFlags[i] != 0) {
+ callableTypeBuilder.SetArgumentFlags(argFlags[i]);
+ }
+ }
+
+ callableTypeBuilder.SetOptionalArgs(optionalCount);
if (!payload.empty()) {
- callableTypeBuilder.SetPayload(payload);
- }
- return callableTypeBuilder.Build();
- }
- TMaybe<TType> LoadVariantType(TType underlyingType, ui32 /*level*/) {
+ callableTypeBuilder.SetPayload(payload);
+ }
+ return callableTypeBuilder.Build();
+ }
+ TMaybe<TType> LoadVariantType(TType underlyingType, ui32 /*level*/) {
return Builder.NewVariantType(underlyingType);
}
void Error(const TString& info) {
@@ -292,61 +292,61 @@ struct TRuntimeTypeLoader {
}
};
-NKikimr::NMiniKQL::TType* ParseTypeFromYson(const TStringBuf yson, NKikimr::NMiniKQL::TProgramBuilder& builder, IOutputStream& err) {
+NKikimr::NMiniKQL::TType* ParseTypeFromYson(const TStringBuf yson, NKikimr::NMiniKQL::TProgramBuilder& builder, IOutputStream& err) {
NYT::TNode node;
if (!ParseYson(node, yson, err)) {
return nullptr;
}
TRuntimeTypeLoader loader(builder, err);
- return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr);
+ return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr);
}
NKikimr::NMiniKQL::TType* ParseTypeFromYson(const NYT::TNode& node, NKikimr::NMiniKQL::TProgramBuilder& builder, IOutputStream& err) {
TRuntimeTypeLoader loader(builder, err);
- return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr);
-}
-
-struct TOrderAwareRuntimeTypeLoader: public TRuntimeTypeLoader {
- typedef NKikimr::NMiniKQL::TType* TType;
-
- NCommon::TCodecContext& Ctx;
-
- TOrderAwareRuntimeTypeLoader(NCommon::TCodecContext& ctx, IOutputStream& err)
- : TRuntimeTypeLoader(ctx.Builder, err)
- , Ctx(ctx)
- {
- }
- TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 level) {
- auto type = Builder.NewEmptyStructType();
- bool sorted = true;
- TString prev;
- for (auto& member : members) {
- if (member.first < prev) {
- sorted = false;
- }
- prev = member.first;
- type = Builder.NewStructType(type, member.first, member.second);
- }
- if (level > 0 && !sorted) {
- std::vector<size_t> reorder(members.size(), 0);
- using namespace NKikimr::NMiniKQL;
- auto structType = AS_TYPE(TStructType, type);
- size_t i = 0;
- for (auto& member : members) {
- reorder[i++] = structType->GetMemberIndex(member.first);
- }
- Ctx.StructReorders.push_back(std::move(reorder));
- type->SetCookie((ui64)&Ctx.StructReorders.back());
- }
- return type;
- }
-};
-
-NKikimr::NMiniKQL::TType* ParseOrderAwareTypeFromYson(const NYT::TNode& node, TCodecContext& ctx, IOutputStream& err) {
- TOrderAwareRuntimeTypeLoader loader(ctx, err);
- return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr);
+ return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr);
}
+struct TOrderAwareRuntimeTypeLoader: public TRuntimeTypeLoader {
+ typedef NKikimr::NMiniKQL::TType* TType;
+
+ NCommon::TCodecContext& Ctx;
+
+ TOrderAwareRuntimeTypeLoader(NCommon::TCodecContext& ctx, IOutputStream& err)
+ : TRuntimeTypeLoader(ctx.Builder, err)
+ , Ctx(ctx)
+ {
+ }
+ TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 level) {
+ auto type = Builder.NewEmptyStructType();
+ bool sorted = true;
+ TString prev;
+ for (auto& member : members) {
+ if (member.first < prev) {
+ sorted = false;
+ }
+ prev = member.first;
+ type = Builder.NewStructType(type, member.first, member.second);
+ }
+ if (level > 0 && !sorted) {
+ std::vector<size_t> reorder(members.size(), 0);
+ using namespace NKikimr::NMiniKQL;
+ auto structType = AS_TYPE(TStructType, type);
+ size_t i = 0;
+ for (auto& member : members) {
+ reorder[i++] = structType->GetMemberIndex(member.first);
+ }
+ Ctx.StructReorders.push_back(std::move(reorder));
+ type->SetCookie((ui64)&Ctx.StructReorders.back());
+ }
+ return type;
+ }
+};
+
+NKikimr::NMiniKQL::TType* ParseOrderAwareTypeFromYson(const NYT::TNode& node, TCodecContext& ctx, IOutputStream& err) {
+ TOrderAwareRuntimeTypeLoader loader(ctx, err);
+ return DoLoadTypeFromYson(loader, node, 0).GetOrElse(nullptr);
+}
+
} // namespace NCommon
} // namespace NYql
diff --git a/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h b/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h
index 9a21a1f388..02f8218ac9 100644
--- a/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h
+++ b/ydb/library/yql/providers/common/schema/mkql/yql_mkql_schema.h
@@ -3,11 +3,11 @@
#include <library/cpp/yson/consumer.h>
#include <library/cpp/yson/node/node.h>
-#include <util/stream/output.h>
-#include <util/generic/strbuf.h>
-
-#include <functional>
-
+#include <util/stream/output.h>
+#include <util/generic/strbuf.h>
+
+#include <functional>
+
namespace NKikimr {
namespace NMiniKQL {
@@ -21,15 +21,15 @@ class TProgramBuilder;
namespace NYql {
namespace NCommon {
-struct TCodecContext;
-
+struct TCodecContext;
+
void WriteTypeToYson(NYson::TYsonConsumerBase& writer, const NKikimr::NMiniKQL::TType* type);
NYT::TNode TypeToYsonNode(const NKikimr::NMiniKQL::TType* type);
TString WriteTypeToYson(const NKikimr::NMiniKQL::TType* type, NYT::NYson::EYsonFormat format = NYT::NYson::EYsonFormat::Binary);
-NKikimr::NMiniKQL::TType* ParseTypeFromYson(const TStringBuf yson, NKikimr::NMiniKQL::TProgramBuilder& builder, IOutputStream& err);
+NKikimr::NMiniKQL::TType* ParseTypeFromYson(const TStringBuf yson, NKikimr::NMiniKQL::TProgramBuilder& builder, IOutputStream& err);
NKikimr::NMiniKQL::TType* ParseTypeFromYson(const NYT::TNode& node, NKikimr::NMiniKQL::TProgramBuilder& builder, IOutputStream& err);
-NKikimr::NMiniKQL::TType* ParseOrderAwareTypeFromYson(const NYT::TNode& node, TCodecContext& ctx, IOutputStream& err);
+NKikimr::NMiniKQL::TType* ParseOrderAwareTypeFromYson(const NYT::TNode& node, TCodecContext& ctx, IOutputStream& err);
} // namespace NCommon
} // namespace NYql
diff --git a/ydb/library/yql/providers/common/schema/parser/ya.make b/ydb/library/yql/providers/common/schema/parser/ya.make
index 77b56b6c16..e5f785846d 100644
--- a/ydb/library/yql/providers/common/schema/parser/ya.make
+++ b/ydb/library/yql/providers/common/schema/parser/ya.make
@@ -1,14 +1,14 @@
-LIBRARY()
-
-OWNER(g:yql g:yql_ydb_core)
-
-SRCS(
- yql_type_parser.cpp
-)
-
-PEERDIR(
- library/cpp/yson/node
- library/cpp/yson
-)
-
-END()
+LIBRARY()
+
+OWNER(g:yql g:yql_ydb_core)
+
+SRCS(
+ yql_type_parser.cpp
+)
+
+PEERDIR(
+ library/cpp/yson/node
+ library/cpp/yson
+)
+
+END()
diff --git a/ydb/library/yql/providers/common/schema/parser/yql_type_parser.cpp b/ydb/library/yql/providers/common/schema/parser/yql_type_parser.cpp
index ae3f281ed2..ea6261f720 100644
--- a/ydb/library/yql/providers/common/schema/parser/yql_type_parser.cpp
+++ b/ydb/library/yql/providers/common/schema/parser/yql_type_parser.cpp
@@ -1,58 +1,58 @@
-#include "yql_type_parser.h"
+#include "yql_type_parser.h"
#include <library/cpp/yson/node/node_io.h>
namespace NYql {
namespace NCommon {
-void TYqlTypeYsonSaverBase::SaveTypeHeader(TStringBuf name) {
- Writer.OnBeginList();
- Writer.OnListItem();
- Writer.OnStringScalar(name);
-}
+void TYqlTypeYsonSaverBase::SaveTypeHeader(TStringBuf name) {
+ Writer.OnBeginList();
+ Writer.OnListItem();
+ Writer.OnStringScalar(name);
+}
#define SAVE_TYPE_IMPL(type) \
-void TYqlTypeYsonSaverBase::Save ## type() { \
- SaveTypeHeader(#type); \
- Writer.OnEndList(); \
-}
+void TYqlTypeYsonSaverBase::Save ## type() { \
+ SaveTypeHeader(#type); \
+ Writer.OnEndList(); \
+}
-SAVE_TYPE_IMPL(Type)
-SAVE_TYPE_IMPL(VoidType)
-SAVE_TYPE_IMPL(NullType)
-SAVE_TYPE_IMPL(UnitType)
-SAVE_TYPE_IMPL(GenericType)
-SAVE_TYPE_IMPL(EmptyListType)
-SAVE_TYPE_IMPL(EmptyDictType)
+SAVE_TYPE_IMPL(Type)
+SAVE_TYPE_IMPL(VoidType)
+SAVE_TYPE_IMPL(NullType)
+SAVE_TYPE_IMPL(UnitType)
+SAVE_TYPE_IMPL(GenericType)
+SAVE_TYPE_IMPL(EmptyListType)
+SAVE_TYPE_IMPL(EmptyDictType)
#undef SAVE_TYPE_IMPL
-void TYqlTypeYsonSaverBase::SaveDataType(const TStringBuf& dataType) {
- SaveTypeHeader("DataType");
- Writer.OnListItem();
- Writer.OnStringScalar(dataType);
- Writer.OnEndList();
+void TYqlTypeYsonSaverBase::SaveDataType(const TStringBuf& dataType) {
+ SaveTypeHeader("DataType");
+ Writer.OnListItem();
+ Writer.OnStringScalar(dataType);
+ Writer.OnEndList();
}
-void TYqlTypeYsonSaverBase::SaveDataTypeParams(const TStringBuf& dataType, const TStringBuf& paramOne, const TStringBuf& paramTwo) {
- SaveTypeHeader("DataType");
- Writer.OnListItem();
- Writer.OnStringScalar(dataType);
- Writer.OnListItem();
- Writer.OnStringScalar(paramOne);
- Writer.OnListItem();
- Writer.OnStringScalar(paramTwo);
- Writer.OnEndList();
+void TYqlTypeYsonSaverBase::SaveDataTypeParams(const TStringBuf& dataType, const TStringBuf& paramOne, const TStringBuf& paramTwo) {
+ SaveTypeHeader("DataType");
+ Writer.OnListItem();
+ Writer.OnStringScalar(dataType);
+ Writer.OnListItem();
+ Writer.OnStringScalar(paramOne);
+ Writer.OnListItem();
+ Writer.OnStringScalar(paramTwo);
+ Writer.OnEndList();
}
-void TYqlTypeYsonSaverBase::SaveResourceType(const TStringBuf& tag) {
- SaveTypeHeader("ResourceType");
- Writer.OnListItem();
- Writer.OnStringScalar(tag);
- Writer.OnEndList();
+void TYqlTypeYsonSaverBase::SaveResourceType(const TStringBuf& tag) {
+ SaveTypeHeader("ResourceType");
+ Writer.OnListItem();
+ Writer.OnStringScalar(tag);
+ Writer.OnEndList();
}
-bool ParseYson(NYT::TNode& res, const TStringBuf yson, IOutputStream& err) {
+bool ParseYson(NYT::TNode& res, const TStringBuf yson, IOutputStream& err) {
try {
res = NYT::NodeFromYsonString(yson);
}
diff --git a/ydb/library/yql/providers/common/schema/parser/yql_type_parser.h b/ydb/library/yql/providers/common/schema/parser/yql_type_parser.h
index a3116efb14..cf58a414ac 100644
--- a/ydb/library/yql/providers/common/schema/parser/yql_type_parser.h
+++ b/ydb/library/yql/providers/common/schema/parser/yql_type_parser.h
@@ -3,418 +3,418 @@
#include <library/cpp/yson/consumer.h>
#include <library/cpp/yson/node/node.h>
-#include <util/generic/maybe.h>
-#include <util/generic/string.h>
-#include <util/generic/vector.h>
-#include <util/generic/strbuf.h>
-#include <util/stream/output.h>
-
-namespace NYql {
-namespace NCommon {
-
-class TYqlTypeYsonSaverBase {
-public:
+#include <util/generic/maybe.h>
+#include <util/generic/string.h>
+#include <util/generic/vector.h>
+#include <util/generic/strbuf.h>
+#include <util/stream/output.h>
+
+namespace NYql {
+namespace NCommon {
+
+class TYqlTypeYsonSaverBase {
+public:
typedef NYson::TYsonConsumerBase TConsumer;
- TYqlTypeYsonSaverBase(TConsumer& writer)
- : Writer(writer)
- {
- }
-
-protected:
- void SaveTypeHeader(TStringBuf name);
- void SaveType();
- void SaveVoidType();
- void SaveNullType();
- void SaveUnitType();
- void SaveGenericType();
- void SaveEmptyListType();
- void SaveEmptyDictType();
- void SaveDataType(const TStringBuf& dataType);
- void SaveDataTypeParams(const TStringBuf& dataType, const TStringBuf& paramOne, const TStringBuf& paramTwo);
- void SaveResourceType(const TStringBuf& tag);
-
-protected:
+ TYqlTypeYsonSaverBase(TConsumer& writer)
+ : Writer(writer)
+ {
+ }
+
+protected:
+ void SaveTypeHeader(TStringBuf name);
+ void SaveType();
+ void SaveVoidType();
+ void SaveNullType();
+ void SaveUnitType();
+ void SaveGenericType();
+ void SaveEmptyListType();
+ void SaveEmptyDictType();
+ void SaveDataType(const TStringBuf& dataType);
+ void SaveDataTypeParams(const TStringBuf& dataType, const TStringBuf& paramOne, const TStringBuf& paramTwo);
+ void SaveResourceType(const TStringBuf& tag);
+
+protected:
NYson::TYsonConsumerBase& Writer;
-};
-
-
-template <typename TDerived>
-class TYqlTypeYsonSaverImpl: public TYqlTypeYsonSaverBase {
- typedef TYqlTypeYsonSaverImpl<TDerived> TSelf;
-
-public:
- TYqlTypeYsonSaverImpl(TConsumer& writer)
- : TYqlTypeYsonSaverBase(writer)
- {
- }
-
- template <typename TType>
- void Save(TType* type) {
- static_cast<TDerived*>(this)->Save(type);
- }
-
-protected:
- template <typename TTaggedType>
- void SaveTaggedType(const TTaggedType& taggedType) {
- SaveTypeHeader("TaggedType");
- Writer.OnListItem();
- Writer.OnStringScalar(taggedType.GetTag());
- Writer.OnListItem();
- TSelf baseType(Writer);
- baseType.Save(taggedType.GetBaseType());
- Writer.OnEndList();
- }
-
- template <typename TStructType>
- void SaveStructType(const TStructType& structType) {
- SaveTypeHeader("StructType");
- Writer.OnListItem();
- Writer.OnBeginList();
- for (ui32 i = 0, e = structType.GetMembersCount(); i < e; ++i) {
- Writer.OnListItem();
- Writer.OnBeginList();
- Writer.OnListItem();
- Writer.OnStringScalar(structType.GetMemberName(i));
- Writer.OnListItem();
- TSelf value(Writer);
- value.Save(structType.GetMemberType(i));
- Writer.OnEndList();
- }
- Writer.OnEndList();
- Writer.OnEndList();
- }
-
- template <typename TListType>
- void SaveListType(const TListType& listType) {
- SaveTypeHeader("ListType");
- Writer.OnListItem();
- TSelf item(Writer);
- item.Save(listType.GetItemType());
- Writer.OnEndList();
- }
-
- template <typename TStreamType>
- void SaveStreamType(const TStreamType& streamType) {
- SaveTypeHeader("StreamType");
- Writer.OnListItem();
- TSelf item(Writer);
- item.Save(streamType.GetItemType());
- Writer.OnEndList();
- }
-
- template <typename TOptionalType>
- void SaveOptionalType(const TOptionalType& optionalType) {
- SaveTypeHeader("OptionalType");
- Writer.OnListItem();
- TSelf item(Writer);
- item.Save(optionalType.GetItemType());
- Writer.OnEndList();
- }
-
- template <typename TDictType>
- void SaveDictType(const TDictType& dictType) {
- SaveTypeHeader("DictType");
- Writer.OnListItem();
- TSelf key(Writer);
- key.Save(dictType.GetKeyType());
- Writer.OnListItem();
- TSelf val(Writer);
- val.Save(dictType.GetPayloadType());
- Writer.OnEndList();
- }
-
- template <typename TTupleType>
- void SaveTupleType(const TTupleType& tupleType) {
- SaveTypeHeader("TupleType");
- Writer.OnListItem();
- Writer.OnBeginList();
- for (ui32 i = 0, e = tupleType.GetElementsCount(); i < e; ++i) {
- Writer.OnListItem();
- TSelf element(Writer);
- element.Save(tupleType.GetElementType(i));
- }
- Writer.OnEndList();
- Writer.OnEndList();
- }
-
- template <typename TCallableType>
- void SaveCallableType(const TCallableType& callableType) {
- SaveTypeHeader("CallableType");
- Writer.OnListItem();
- // main settings
- Writer.OnBeginList();
- if (callableType.GetOptionalArgsCount() > 0 || !callableType.GetPayload().empty()) {
- Writer.OnListItem();
- Writer.OnUint64Scalar(callableType.GetOptionalArgsCount());
- }
-
- if (!callableType.GetPayload().empty()) {
- Writer.OnListItem();
- Writer.OnStringScalar(callableType.GetPayload());
- }
-
- Writer.OnEndList();
- // ret
- Writer.OnListItem();
- Writer.OnBeginList();
- Writer.OnListItem();
- TSelf ret(Writer);
- ret.Save(callableType.GetReturnType());
- Writer.OnEndList();
- // args
- Writer.OnListItem();
- Writer.OnBeginList();
- for (ui32 i = 0, e = callableType.GetArgumentsCount(); i < e; ++i) {
- Writer.OnListItem();
- Writer.OnBeginList();
- Writer.OnListItem();
- TSelf arg(Writer);
- arg.Save(callableType.GetArgumentType(i));
- if (!callableType.GetArgumentName(i).empty()) {
- Writer.OnListItem();
- Writer.OnStringScalar(callableType.GetArgumentName(i));
- }
-
- if (callableType.GetArgumentFlags(i) != 0) {
- Writer.OnListItem();
- Writer.OnUint64Scalar(callableType.GetArgumentFlags(i));
- }
-
- Writer.OnEndList();
- }
-
- Writer.OnEndList();
- Writer.OnEndList();
- }
-
- template <typename TVariantType>
- void SaveVariantType(const TVariantType& variantType) {
- SaveTypeHeader("VariantType");
- Writer.OnListItem();
- TSelf item(Writer);
- item.Save(variantType.GetUnderlyingType());
- Writer.OnEndList();
- }
-};
-
-template <typename TLoader>
-TMaybe<typename TLoader::TType> DoLoadTypeFromYson(TLoader& loader, const NYT::TNode& node, ui32 level) {
- if (!node.IsList() || node.Size() < 1 || !node[0].IsString()) {
- loader.Error("Invalid type scheme");
- return Nothing();
- }
- auto typeName = node[0].AsString();
- if (typeName == "VoidType") {
- return loader.LoadVoidType(level);
- } else if (typeName == "NullType") {
- return loader.LoadNullType(level);
- } else if (typeName == "UnitType") {
- return loader.LoadUnitType(level);
- } else if (typeName == "GenericType") {
- return loader.LoadGenericType(level);
- } else if (typeName == "EmptyListType") {
- return loader.LoadEmptyListType(level);
- } else if (typeName == "EmptyDictType") {
- return loader.LoadEmptyDictType(level);
- } else if (typeName == "DataType") {
- if ((node.Size() != 2 && node.Size() != 4) || !node[1].IsString()) {
- loader.Error("Invalid data type scheme");
- return Nothing();
- }
- if (node.Size() == 2) {
- return loader.LoadDataType(node[1].AsString(), level);
- }
-
- if (!node[2].IsString() || !node[3].IsString()) {
- loader.Error("Invalid data type scheme");
- return Nothing();
- }
-
- return loader.LoadDataTypeParams(node[1].AsString(), node[2].AsString(), node[3].AsString(), level);
- } else if (typeName == "ResourceType") {
- if (node.Size() != 2 || !node[1].IsString()) {
- loader.Error("Invalid resource type scheme");
- return Nothing();
- }
- return loader.LoadResourceType(node[1].AsString(), level);
- } else if (typeName == "TaggedType") {
- if (node.Size() != 3 || !node[1].IsString()) {
- loader.Error("Invalid tagged type scheme");
- return Nothing();
- }
- auto baseType = DoLoadTypeFromYson(loader, node[2], level); // Don't increase level type for tagged type
- if (!baseType) {
- return Nothing();
- }
- return loader.LoadTaggedType(*baseType, node[1].AsString(), level);
- } else if (typeName == "ErrorType") {
- if (node.Size() != 5 || !node[1].IsInt64() || !node[2].IsInt64() || !node[3].IsString() || !node[4].IsString()) {
- loader.Error("Invalid error type scheme");
- return Nothing();
- }
- return loader.LoadErrorType(node[1].AsInt64(), node[2].AsInt64(), node[3].AsString(), node[4].AsString(), level);
- } else if (typeName == "StructType") {
- if (node.Size() != 2 || !node[1].IsList()) {
- loader.Error("Invalid struct type scheme");
- return Nothing();
- }
- TVector<std::pair<TString, typename TLoader::TType>> members;
- for (auto& member : node[1].AsList()) {
- if (!member.IsList() || member.Size() != 2 || !member[0].IsString()) {
- loader.Error("Invalid struct type scheme");
- return Nothing();
- }
-
- auto name = member[0].AsString();
- auto memberType = DoLoadTypeFromYson(loader, member[1], level + 1);
- if (!memberType) {
- return Nothing();
- }
- members.push_back(std::make_pair(name, *memberType));
- }
- return loader.LoadStructType(members, level);
- } else if (typeName == "ListType") {
- if (node.Size() != 2) {
- loader.Error("Invalid list type scheme");
- return Nothing();
- }
- auto itemType = DoLoadTypeFromYson(loader, node[1], level + 1);
- if (!itemType) {
- return Nothing();
- }
- return loader.LoadListType(*itemType, level);
- } else if (typeName == "StreamType") {
- if (node.Size() != 2) {
- loader.Error("Invalid list type scheme");
- return Nothing();
- }
- auto itemType = DoLoadTypeFromYson(loader, node[1], level + 1);
- if (!itemType) {
- return Nothing();
- }
- return loader.LoadStreamType(*itemType, level);
-
- } else if (typeName == "OptionalType") {
- if (node.Size() != 2) {
- loader.Error("Invalid optional type scheme");
- return Nothing();
- }
- auto itemType = DoLoadTypeFromYson(loader, node[1], level + 1);
- if (!itemType) {
- return Nothing();
- }
- return loader.LoadOptionalType(*itemType, level);
- } else if (typeName == "TupleType") {
- if (node.Size() != 2 || !node[1].IsList()) {
- loader.Error("Invalid tuple type scheme");
- return Nothing();
- }
- TVector<typename TLoader::TType> elements;
- for (auto& item: node[1].AsList()) {
- auto itemType = DoLoadTypeFromYson(loader, item, level + 1);
- if (!itemType) {
- return Nothing();
- }
- elements.push_back(*itemType);
- }
- return loader.LoadTupleType(elements, level);
- } else if (typeName == "DictType") {
- if (node.Size() != 3) {
- loader.Error("Invalid dict type scheme");
- return Nothing();
- }
- auto keyType = DoLoadTypeFromYson(loader, node[1], level + 1);
- auto valType = DoLoadTypeFromYson(loader, node[2], level + 1);
- if (!keyType || !valType) {
- return Nothing();
- }
- return loader.LoadDictType(*keyType, *valType, level);
- } else if (typeName == "CallableType") {
- if (node.Size() != 4 || !node[1].IsList() || !node[2].IsList() || !node[3].IsList()) {
- loader.Error("Invalid callable type scheme");
- return Nothing();
- }
- ui32 optionalCount = 0;
- TString payload;
- if (!node[1].AsList().empty()) {
- auto& list = node[1].AsList();
- if (!list[0].IsUint64()) {
- loader.Error("Invalid callable type scheme");
- return Nothing();
- }
- optionalCount = list[0].AsUint64();
- if (list.size() > 1) {
- if (!list[1].IsString()) {
- loader.Error("Invalid callable type scheme");
- return Nothing();
- }
- payload = list[1].AsString();
- }
- if (list.size() > 2) {
- loader.Error("Invalid callable type scheme");
- return Nothing();
- }
- }
-
- if (node[2].AsList().size() != 1) {
- loader.Error("Invalid callable type scheme");
- return Nothing();
- }
- auto returnType = DoLoadTypeFromYson(loader, node[2].AsList()[0], level + 1);
- if (!returnType) {
- return Nothing();
- }
-
- TVector<typename TLoader::TType> argTypes;
- TVector<TString> argNames;
- TVector<ui64> argFlags;
- for (auto& item: node[3].AsList()) {
- if (!item.IsList() || item.AsList().size() < 1 || item.AsList().size() > 3) {
- loader.Error("Invalid callable type scheme");
- return Nothing();
- }
-
- auto argType = DoLoadTypeFromYson(loader, item.AsList()[0], level + 1);
- if (!argType) {
- return Nothing();
- }
- argTypes.push_back(*argType);
- if (item.AsList().size() > 1 && item.AsList()[1].IsString()) {
- argNames.push_back(item.AsList()[1].AsString());
- } else {
- argNames.emplace_back();
- }
- if (item.AsList().size() > 1 && item.AsList()[1].IsUint64()) {
- argFlags.push_back(item.AsList()[1].AsUint64());
- } else if (item.AsList().size() > 2) {
- if (!item.AsList()[2].IsUint64()) {
- loader.Error("Invalid callable type scheme");
- return Nothing();
- }
- argFlags.push_back(item.AsList()[2].AsUint64());
- } else {
- argFlags.emplace_back();
- }
- }
-
- return loader.LoadCallableType(*returnType, argTypes, argNames, argFlags, optionalCount, payload, level);
- } else if (typeName == "VariantType") {
- if (node.Size() != 2) {
- loader.Error("Invalid variant type scheme");
- return Nothing();
- }
- auto underlyingType = DoLoadTypeFromYson(loader, node[1], level + 1);
- if (!underlyingType) {
- return Nothing();
- }
- return loader.LoadVariantType(*underlyingType, level);
- }
- loader.Error("unsupported type: " + typeName);
- return Nothing();
-}
-
-bool ParseYson(NYT::TNode& res, const TStringBuf yson, IOutputStream& err);
+};
+
+
+template <typename TDerived>
+class TYqlTypeYsonSaverImpl: public TYqlTypeYsonSaverBase {
+ typedef TYqlTypeYsonSaverImpl<TDerived> TSelf;
+
+public:
+ TYqlTypeYsonSaverImpl(TConsumer& writer)
+ : TYqlTypeYsonSaverBase(writer)
+ {
+ }
+
+ template <typename TType>
+ void Save(TType* type) {
+ static_cast<TDerived*>(this)->Save(type);
+ }
+
+protected:
+ template <typename TTaggedType>
+ void SaveTaggedType(const TTaggedType& taggedType) {
+ SaveTypeHeader("TaggedType");
+ Writer.OnListItem();
+ Writer.OnStringScalar(taggedType.GetTag());
+ Writer.OnListItem();
+ TSelf baseType(Writer);
+ baseType.Save(taggedType.GetBaseType());
+ Writer.OnEndList();
+ }
+
+ template <typename TStructType>
+ void SaveStructType(const TStructType& structType) {
+ SaveTypeHeader("StructType");
+ Writer.OnListItem();
+ Writer.OnBeginList();
+ for (ui32 i = 0, e = structType.GetMembersCount(); i < e; ++i) {
+ Writer.OnListItem();
+ Writer.OnBeginList();
+ Writer.OnListItem();
+ Writer.OnStringScalar(structType.GetMemberName(i));
+ Writer.OnListItem();
+ TSelf value(Writer);
+ value.Save(structType.GetMemberType(i));
+ Writer.OnEndList();
+ }
+ Writer.OnEndList();
+ Writer.OnEndList();
+ }
+
+ template <typename TListType>
+ void SaveListType(const TListType& listType) {
+ SaveTypeHeader("ListType");
+ Writer.OnListItem();
+ TSelf item(Writer);
+ item.Save(listType.GetItemType());
+ Writer.OnEndList();
+ }
+
+ template <typename TStreamType>
+ void SaveStreamType(const TStreamType& streamType) {
+ SaveTypeHeader("StreamType");
+ Writer.OnListItem();
+ TSelf item(Writer);
+ item.Save(streamType.GetItemType());
+ Writer.OnEndList();
+ }
+
+ template <typename TOptionalType>
+ void SaveOptionalType(const TOptionalType& optionalType) {
+ SaveTypeHeader("OptionalType");
+ Writer.OnListItem();
+ TSelf item(Writer);
+ item.Save(optionalType.GetItemType());
+ Writer.OnEndList();
+ }
+
+ template <typename TDictType>
+ void SaveDictType(const TDictType& dictType) {
+ SaveTypeHeader("DictType");
+ Writer.OnListItem();
+ TSelf key(Writer);
+ key.Save(dictType.GetKeyType());
+ Writer.OnListItem();
+ TSelf val(Writer);
+ val.Save(dictType.GetPayloadType());
+ Writer.OnEndList();
+ }
+
+ template <typename TTupleType>
+ void SaveTupleType(const TTupleType& tupleType) {
+ SaveTypeHeader("TupleType");
+ Writer.OnListItem();
+ Writer.OnBeginList();
+ for (ui32 i = 0, e = tupleType.GetElementsCount(); i < e; ++i) {
+ Writer.OnListItem();
+ TSelf element(Writer);
+ element.Save(tupleType.GetElementType(i));
+ }
+ Writer.OnEndList();
+ Writer.OnEndList();
+ }
+
+ template <typename TCallableType>
+ void SaveCallableType(const TCallableType& callableType) {
+ SaveTypeHeader("CallableType");
+ Writer.OnListItem();
+ // main settings
+ Writer.OnBeginList();
+ if (callableType.GetOptionalArgsCount() > 0 || !callableType.GetPayload().empty()) {
+ Writer.OnListItem();
+ Writer.OnUint64Scalar(callableType.GetOptionalArgsCount());
+ }
+
+ if (!callableType.GetPayload().empty()) {
+ Writer.OnListItem();
+ Writer.OnStringScalar(callableType.GetPayload());
+ }
+
+ Writer.OnEndList();
+ // ret
+ Writer.OnListItem();
+ Writer.OnBeginList();
+ Writer.OnListItem();
+ TSelf ret(Writer);
+ ret.Save(callableType.GetReturnType());
+ Writer.OnEndList();
+ // args
+ Writer.OnListItem();
+ Writer.OnBeginList();
+ for (ui32 i = 0, e = callableType.GetArgumentsCount(); i < e; ++i) {
+ Writer.OnListItem();
+ Writer.OnBeginList();
+ Writer.OnListItem();
+ TSelf arg(Writer);
+ arg.Save(callableType.GetArgumentType(i));
+ if (!callableType.GetArgumentName(i).empty()) {
+ Writer.OnListItem();
+ Writer.OnStringScalar(callableType.GetArgumentName(i));
+ }
+
+ if (callableType.GetArgumentFlags(i) != 0) {
+ Writer.OnListItem();
+ Writer.OnUint64Scalar(callableType.GetArgumentFlags(i));
+ }
+
+ Writer.OnEndList();
+ }
+
+ Writer.OnEndList();
+ Writer.OnEndList();
+ }
+
+ template <typename TVariantType>
+ void SaveVariantType(const TVariantType& variantType) {
+ SaveTypeHeader("VariantType");
+ Writer.OnListItem();
+ TSelf item(Writer);
+ item.Save(variantType.GetUnderlyingType());
+ Writer.OnEndList();
+ }
+};
+
+template <typename TLoader>
+TMaybe<typename TLoader::TType> DoLoadTypeFromYson(TLoader& loader, const NYT::TNode& node, ui32 level) {
+ if (!node.IsList() || node.Size() < 1 || !node[0].IsString()) {
+ loader.Error("Invalid type scheme");
+ return Nothing();
+ }
+ auto typeName = node[0].AsString();
+ if (typeName == "VoidType") {
+ return loader.LoadVoidType(level);
+ } else if (typeName == "NullType") {
+ return loader.LoadNullType(level);
+ } else if (typeName == "UnitType") {
+ return loader.LoadUnitType(level);
+ } else if (typeName == "GenericType") {
+ return loader.LoadGenericType(level);
+ } else if (typeName == "EmptyListType") {
+ return loader.LoadEmptyListType(level);
+ } else if (typeName == "EmptyDictType") {
+ return loader.LoadEmptyDictType(level);
+ } else if (typeName == "DataType") {
+ if ((node.Size() != 2 && node.Size() != 4) || !node[1].IsString()) {
+ loader.Error("Invalid data type scheme");
+ return Nothing();
+ }
+ if (node.Size() == 2) {
+ return loader.LoadDataType(node[1].AsString(), level);
+ }
+
+ if (!node[2].IsString() || !node[3].IsString()) {
+ loader.Error("Invalid data type scheme");
+ return Nothing();
+ }
+
+ return loader.LoadDataTypeParams(node[1].AsString(), node[2].AsString(), node[3].AsString(), level);
+ } else if (typeName == "ResourceType") {
+ if (node.Size() != 2 || !node[1].IsString()) {
+ loader.Error("Invalid resource type scheme");
+ return Nothing();
+ }
+ return loader.LoadResourceType(node[1].AsString(), level);
+ } else if (typeName == "TaggedType") {
+ if (node.Size() != 3 || !node[1].IsString()) {
+ loader.Error("Invalid tagged type scheme");
+ return Nothing();
+ }
+ auto baseType = DoLoadTypeFromYson(loader, node[2], level); // Don't increase level type for tagged type
+ if (!baseType) {
+ return Nothing();
+ }
+ return loader.LoadTaggedType(*baseType, node[1].AsString(), level);
+ } else if (typeName == "ErrorType") {
+ if (node.Size() != 5 || !node[1].IsInt64() || !node[2].IsInt64() || !node[3].IsString() || !node[4].IsString()) {
+ loader.Error("Invalid error type scheme");
+ return Nothing();
+ }
+ return loader.LoadErrorType(node[1].AsInt64(), node[2].AsInt64(), node[3].AsString(), node[4].AsString(), level);
+ } else if (typeName == "StructType") {
+ if (node.Size() != 2 || !node[1].IsList()) {
+ loader.Error("Invalid struct type scheme");
+ return Nothing();
+ }
+ TVector<std::pair<TString, typename TLoader::TType>> members;
+ for (auto& member : node[1].AsList()) {
+ if (!member.IsList() || member.Size() != 2 || !member[0].IsString()) {
+ loader.Error("Invalid struct type scheme");
+ return Nothing();
+ }
+
+ auto name = member[0].AsString();
+ auto memberType = DoLoadTypeFromYson(loader, member[1], level + 1);
+ if (!memberType) {
+ return Nothing();
+ }
+ members.push_back(std::make_pair(name, *memberType));
+ }
+ return loader.LoadStructType(members, level);
+ } else if (typeName == "ListType") {
+ if (node.Size() != 2) {
+ loader.Error("Invalid list type scheme");
+ return Nothing();
+ }
+ auto itemType = DoLoadTypeFromYson(loader, node[1], level + 1);
+ if (!itemType) {
+ return Nothing();
+ }
+ return loader.LoadListType(*itemType, level);
+ } else if (typeName == "StreamType") {
+ if (node.Size() != 2) {
+ loader.Error("Invalid list type scheme");
+ return Nothing();
+ }
+ auto itemType = DoLoadTypeFromYson(loader, node[1], level + 1);
+ if (!itemType) {
+ return Nothing();
+ }
+ return loader.LoadStreamType(*itemType, level);
+
+ } else if (typeName == "OptionalType") {
+ if (node.Size() != 2) {
+ loader.Error("Invalid optional type scheme");
+ return Nothing();
+ }
+ auto itemType = DoLoadTypeFromYson(loader, node[1], level + 1);
+ if (!itemType) {
+ return Nothing();
+ }
+ return loader.LoadOptionalType(*itemType, level);
+ } else if (typeName == "TupleType") {
+ if (node.Size() != 2 || !node[1].IsList()) {
+ loader.Error("Invalid tuple type scheme");
+ return Nothing();
+ }
+ TVector<typename TLoader::TType> elements;
+ for (auto& item: node[1].AsList()) {
+ auto itemType = DoLoadTypeFromYson(loader, item, level + 1);
+ if (!itemType) {
+ return Nothing();
+ }
+ elements.push_back(*itemType);
+ }
+ return loader.LoadTupleType(elements, level);
+ } else if (typeName == "DictType") {
+ if (node.Size() != 3) {
+ loader.Error("Invalid dict type scheme");
+ return Nothing();
+ }
+ auto keyType = DoLoadTypeFromYson(loader, node[1], level + 1);
+ auto valType = DoLoadTypeFromYson(loader, node[2], level + 1);
+ if (!keyType || !valType) {
+ return Nothing();
+ }
+ return loader.LoadDictType(*keyType, *valType, level);
+ } else if (typeName == "CallableType") {
+ if (node.Size() != 4 || !node[1].IsList() || !node[2].IsList() || !node[3].IsList()) {
+ loader.Error("Invalid callable type scheme");
+ return Nothing();
+ }
+ ui32 optionalCount = 0;
+ TString payload;
+ if (!node[1].AsList().empty()) {
+ auto& list = node[1].AsList();
+ if (!list[0].IsUint64()) {
+ loader.Error("Invalid callable type scheme");
+ return Nothing();
+ }
+ optionalCount = list[0].AsUint64();
+ if (list.size() > 1) {
+ if (!list[1].IsString()) {
+ loader.Error("Invalid callable type scheme");
+ return Nothing();
+ }
+ payload = list[1].AsString();
+ }
+ if (list.size() > 2) {
+ loader.Error("Invalid callable type scheme");
+ return Nothing();
+ }
+ }
+
+ if (node[2].AsList().size() != 1) {
+ loader.Error("Invalid callable type scheme");
+ return Nothing();
+ }
+ auto returnType = DoLoadTypeFromYson(loader, node[2].AsList()[0], level + 1);
+ if (!returnType) {
+ return Nothing();
+ }
+
+ TVector<typename TLoader::TType> argTypes;
+ TVector<TString> argNames;
+ TVector<ui64> argFlags;
+ for (auto& item: node[3].AsList()) {
+ if (!item.IsList() || item.AsList().size() < 1 || item.AsList().size() > 3) {
+ loader.Error("Invalid callable type scheme");
+ return Nothing();
+ }
+
+ auto argType = DoLoadTypeFromYson(loader, item.AsList()[0], level + 1);
+ if (!argType) {
+ return Nothing();
+ }
+ argTypes.push_back(*argType);
+ if (item.AsList().size() > 1 && item.AsList()[1].IsString()) {
+ argNames.push_back(item.AsList()[1].AsString());
+ } else {
+ argNames.emplace_back();
+ }
+ if (item.AsList().size() > 1 && item.AsList()[1].IsUint64()) {
+ argFlags.push_back(item.AsList()[1].AsUint64());
+ } else if (item.AsList().size() > 2) {
+ if (!item.AsList()[2].IsUint64()) {
+ loader.Error("Invalid callable type scheme");
+ return Nothing();
+ }
+ argFlags.push_back(item.AsList()[2].AsUint64());
+ } else {
+ argFlags.emplace_back();
+ }
+ }
+
+ return loader.LoadCallableType(*returnType, argTypes, argNames, argFlags, optionalCount, payload, level);
+ } else if (typeName == "VariantType") {
+ if (node.Size() != 2) {
+ loader.Error("Invalid variant type scheme");
+ return Nothing();
+ }
+ auto underlyingType = DoLoadTypeFromYson(loader, node[1], level + 1);
+ if (!underlyingType) {
+ return Nothing();
+ }
+ return loader.LoadVariantType(*underlyingType, level);
+ }
+ loader.Error("unsupported type: " + typeName);
+ return Nothing();
+}
+
+bool ParseYson(NYT::TNode& res, const TStringBuf yson, IOutputStream& err);
} // namespace NCommon
} // namespace NYql
diff --git a/ydb/library/yql/providers/common/schema/skiff/ya.make b/ydb/library/yql/providers/common/schema/skiff/ya.make
index f8da2d013b..4fd01dfa87 100644
--- a/ydb/library/yql/providers/common/schema/skiff/ya.make
+++ b/ydb/library/yql/providers/common/schema/skiff/ya.make
@@ -1,19 +1,19 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(
g:yql
g:yql_ydb_core
)
-
-SRCS(
- yql_skiff_schema.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ yql_skiff_schema.cpp
+)
+
+PEERDIR(
library/cpp/yson/node
ydb/library/yql/public/udf
ydb/library/yql/providers/common/codec
ydb/library/yql/providers/common/schema/parser
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/providers/common/schema/skiff/yql_skiff_schema.cpp b/ydb/library/yql/providers/common/schema/skiff/yql_skiff_schema.cpp
index 2461a17094..348d96d215 100644
--- a/ydb/library/yql/providers/common/schema/skiff/yql_skiff_schema.cpp
+++ b/ydb/library/yql/providers/common/schema/skiff/yql_skiff_schema.cpp
@@ -1,223 +1,223 @@
-#include "yql_skiff_schema.h"
+#include "yql_skiff_schema.h"
#include <ydb/library/yql/providers/common/schema/parser/yql_type_parser.h>
#include <ydb/library/yql/providers/common/codec/yql_codec_type_flags.h>
#include <ydb/library/yql/public/udf/udf_data_type.h>
-
+
#include <library/cpp/yson/node/node_io.h>
-#include <util/generic/yexception.h>
-
+#include <util/generic/yexception.h>
+
namespace NYql {
namespace NCommon {
-struct TSkiffTypeLoader {
- typedef NYT::TNode TType;
-
- TSkiffTypeLoader(ui64 nativeYTTypesFlags)
- : NativeYTTypesFlags(nativeYTTypesFlags)
- {
- }
-
- TMaybe<TType> LoadVoidType(ui32 /*level*/) {
- return NYT::TNode()("wire_type", (NativeYTTypesFlags & NTCF_VOID) ? "nothing" : "yson32");
- }
- TMaybe<TType> LoadNullType(ui32 /*level*/) {
- return NYT::TNode()("wire_type", (NativeYTTypesFlags & NTCF_NULL) ? "nothing" : "yson32");
- }
- TMaybe<TType> LoadUnitType(ui32 /*level*/) {
- ythrow yexception() << "Unsupported type: Unit";
- }
- TMaybe<TType> LoadGenericType(ui32 /*level*/) {
- ythrow yexception() << "Unsupported type: Generic";
- }
- TMaybe<TType> LoadEmptyListType(ui32 /*level*/) {
- return NYT::TNode()("wire_type", (NativeYTTypesFlags & NTCF_COMPLEX) ? "nothing" : "yson32");
- }
- TMaybe<TType> LoadEmptyDictType(ui32 /*level*/) {
- return NYT::TNode()("wire_type", (NativeYTTypesFlags & NTCF_COMPLEX) ? "nothing" : "yson32");
- }
- TMaybe<TType> LoadDataType(const TString& dataType, ui32 /*level*/) {
- const auto slot = NUdf::FindDataSlot(dataType);
- if (!slot) {
- ythrow yexception() << "Unsupported data type: " << dataType;
- }
-
- switch (*slot) {
- case NUdf::EDataSlot::Bool:
- return NYT::TNode()("wire_type", "boolean");
- case NUdf::EDataSlot::Int8:
- case NUdf::EDataSlot::Int16:
- case NUdf::EDataSlot::Int32:
- case NUdf::EDataSlot::Int64:
- case NUdf::EDataSlot::Interval:
- return NYT::TNode()("wire_type", "int64");
- case NUdf::EDataSlot::Uint8:
- case NUdf::EDataSlot::Uint16:
- case NUdf::EDataSlot::Uint32:
- case NUdf::EDataSlot::Uint64:
- case NUdf::EDataSlot::Date:
- case NUdf::EDataSlot::Datetime:
- case NUdf::EDataSlot::Timestamp:
- return NYT::TNode()("wire_type", "uint64");
- case NUdf::EDataSlot::String:
- case NUdf::EDataSlot::Utf8:
- case NUdf::EDataSlot::Json:
- case NUdf::EDataSlot::Uuid:
+struct TSkiffTypeLoader {
+ typedef NYT::TNode TType;
+
+ TSkiffTypeLoader(ui64 nativeYTTypesFlags)
+ : NativeYTTypesFlags(nativeYTTypesFlags)
+ {
+ }
+
+ TMaybe<TType> LoadVoidType(ui32 /*level*/) {
+ return NYT::TNode()("wire_type", (NativeYTTypesFlags & NTCF_VOID) ? "nothing" : "yson32");
+ }
+ TMaybe<TType> LoadNullType(ui32 /*level*/) {
+ return NYT::TNode()("wire_type", (NativeYTTypesFlags & NTCF_NULL) ? "nothing" : "yson32");
+ }
+ TMaybe<TType> LoadUnitType(ui32 /*level*/) {
+ ythrow yexception() << "Unsupported type: Unit";
+ }
+ TMaybe<TType> LoadGenericType(ui32 /*level*/) {
+ ythrow yexception() << "Unsupported type: Generic";
+ }
+ TMaybe<TType> LoadEmptyListType(ui32 /*level*/) {
+ return NYT::TNode()("wire_type", (NativeYTTypesFlags & NTCF_COMPLEX) ? "nothing" : "yson32");
+ }
+ TMaybe<TType> LoadEmptyDictType(ui32 /*level*/) {
+ return NYT::TNode()("wire_type", (NativeYTTypesFlags & NTCF_COMPLEX) ? "nothing" : "yson32");
+ }
+ TMaybe<TType> LoadDataType(const TString& dataType, ui32 /*level*/) {
+ const auto slot = NUdf::FindDataSlot(dataType);
+ if (!slot) {
+ ythrow yexception() << "Unsupported data type: " << dataType;
+ }
+
+ switch (*slot) {
+ case NUdf::EDataSlot::Bool:
+ return NYT::TNode()("wire_type", "boolean");
+ case NUdf::EDataSlot::Int8:
+ case NUdf::EDataSlot::Int16:
+ case NUdf::EDataSlot::Int32:
+ case NUdf::EDataSlot::Int64:
+ case NUdf::EDataSlot::Interval:
+ return NYT::TNode()("wire_type", "int64");
+ case NUdf::EDataSlot::Uint8:
+ case NUdf::EDataSlot::Uint16:
+ case NUdf::EDataSlot::Uint32:
+ case NUdf::EDataSlot::Uint64:
+ case NUdf::EDataSlot::Date:
+ case NUdf::EDataSlot::Datetime:
+ case NUdf::EDataSlot::Timestamp:
+ return NYT::TNode()("wire_type", "uint64");
+ case NUdf::EDataSlot::String:
+ case NUdf::EDataSlot::Utf8:
+ case NUdf::EDataSlot::Json:
+ case NUdf::EDataSlot::Uuid:
case NUdf::EDataSlot::DyNumber:
case NUdf::EDataSlot::JsonDocument:
- return NYT::TNode()("wire_type", "string32");
- case NUdf::EDataSlot::Yson:
- return NYT::TNode()("wire_type", "yson32");
- case NUdf::EDataSlot::Float:
- case NUdf::EDataSlot::Double:
- return NYT::TNode()("wire_type", "double");
- case NUdf::EDataSlot::TzDate:
- case NUdf::EDataSlot::TzDatetime:
- case NUdf::EDataSlot::TzTimestamp:
- return NYT::TNode()("wire_type", "string32");
- case NUdf::EDataSlot::Decimal:
- ythrow yexception() << "Decimal type without parameters.";
- break;
- }
-
- ythrow yexception() << "Unsupported data type" << NUdf::GetDataTypeInfo(*slot).Name;
- }
-
- TMaybe<TType> LoadDataTypeParams(const TString& dataType, const TString& paramOne, const TString& /*paramTwo*/, ui32 /*level*/) {
- const auto slot = NUdf::FindDataSlot(dataType);
- if (!slot) {
- ythrow yexception() << "Unsupported data type: " << dataType;
- }
-
- if (NUdf::EDataSlot::Decimal != slot) {
- ythrow yexception() << "Unexpected parameters for type: " << dataType;
- }
-
- ui8 precision = 0;
- if (!TryFromString(paramOne, precision) || !precision || precision > 35) {
- ythrow yexception() << "Invalid decimal precision: " << paramOne;
- }
-
- if (NativeYTTypesFlags & NTCF_DECIMAL) {
- if (precision < 10) {
- return NYT::TNode()("wire_type", "int32");
- } else if (precision < 19) {
- return NYT::TNode()("wire_type", "int64");
- } else {
- return NYT::TNode()("wire_type", "int128");
- }
- }
-
- return NYT::TNode()("wire_type", "string32");
- }
-
- TMaybe<TType> LoadResourceType(const TString& /*tag*/, ui32 /*level*/) {
- ythrow yexception() << "Unsupported type: Resource";
- }
- TMaybe<TType> LoadTaggedType(TType baseType, const TString& /*tag*/, ui32 /*level*/) {
- return baseType;
- }
- TMaybe<TType> LoadErrorType(ui32 /*row*/, ui32 /*column*/, const TString& /*file*/, const TString& msg, ui32 /*level*/) {
- ythrow yexception() << msg;
- }
- TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 level) {
- if (!(NativeYTTypesFlags & NTCF_COMPLEX) && level > 0) {
- return NYT::TNode()("wire_type", "yson32");
- }
- auto children = NYT::TNode::CreateList();
-
- for (auto& item: members) {
- NYT::TNode innerNode = item.second;
- innerNode["name"] = item.first;
- children.Add(std::move(innerNode));
- }
-
- return NYT::TNode()
- ("wire_type", "tuple")
- ("children", std::move(children));
- }
- TMaybe<TType> LoadListType(TType itemType, ui32 /*level*/) {
- if (!(NativeYTTypesFlags & NTCF_COMPLEX)) {
- return NYT::TNode()("wire_type", "yson32");
- }
- return NYT::TNode()
- ("wire_type", "repeated_variant8")
- ("children", NYT::TNode()
- .Add(std::move(itemType))
- );
- }
- TMaybe<TType> LoadStreamType(TType /*itemType*/, ui32 /*level*/) {
- ythrow yexception() << "Unsupported type: Stream";
- }
- TMaybe<TType> LoadOptionalType(TType itemType, ui32 level) {
- if (!(NativeYTTypesFlags & NTCF_COMPLEX) && level > 1) {
- return NYT::TNode()("wire_type", "yson32");
- }
- return NYT::TNode()
- ("wire_type", "variant8")
- ("children", NYT::TNode()
- .Add(NYT::TNode()("wire_type", "nothing"))
- .Add(std::move(itemType))
- );
- }
- TMaybe<TType> LoadTupleType(const TVector<TType>& elements, ui32 /*level*/) {
- if (!(NativeYTTypesFlags & NTCF_COMPLEX)) {
- return NYT::TNode()("wire_type", "yson32");
- }
-
- auto children = NYT::TNode::CreateList();
- for (auto& inner: elements) {
- children.Add(inner);
- }
-
- return NYT::TNode()
- ("wire_type", "tuple")
- ("children", std::move(children));
- }
- TMaybe<TType> LoadDictType(TType keyType, TType valType, ui32 /*level*/) {
- if (!(NativeYTTypesFlags & NTCF_COMPLEX)) {
- return NYT::TNode()("wire_type", "yson32");
- }
- return NYT::TNode()
- ("wire_type", "repeated_variant8")
- ("children", NYT::TNode()
- .Add(NYT::TNode()
- ("wire_type", "tuple")
- ("children", NYT::TNode()
- .Add(std::move(keyType))
- .Add(std::move(valType))
- )
- )
- );
- }
- TMaybe<TType> LoadCallableType(TType /*returnType*/, const TVector<TType>& /*argTypes*/, const TVector<TString>& /*argNames*/,
- const TVector<ui64>& /*argFlags*/, size_t /*optionalCount*/, const TString& /*payload*/, ui32 /*level*/) {
-
- ythrow yexception() << "Unsupported type: Callable";
- }
- TMaybe<TType> LoadVariantType(TType underlyingType, ui32 /*level*/) {
- if (!(NativeYTTypesFlags & NTCF_COMPLEX)) {
- return NYT::TNode()("wire_type", "yson32");
- }
- if (!underlyingType.IsMap() || !underlyingType.HasKey("children") || !underlyingType["children"].IsList()) {
- ythrow yexception() << "Bad variant underlying type: " << NYT::NodeToYsonString(underlyingType);
- }
- auto altCount = underlyingType["children"].AsList().size();
- underlyingType["wire_type"] = altCount < 256 ? "variant8" : "variant16";
- return underlyingType;
- }
- void Error(const TString& info) {
- ythrow yexception() << info;
- }
-
- ui64 NativeYTTypesFlags;
-};
-
-NYT::TNode ParseSkiffTypeFromYson(const NYT::TNode& node, ui64 nativeYTTypesFlags) {
- TSkiffTypeLoader loader(nativeYTTypesFlags);
- return DoLoadTypeFromYson(loader, node, 0).GetOrElse(NYT::TNode());
-}
-
+ return NYT::TNode()("wire_type", "string32");
+ case NUdf::EDataSlot::Yson:
+ return NYT::TNode()("wire_type", "yson32");
+ case NUdf::EDataSlot::Float:
+ case NUdf::EDataSlot::Double:
+ return NYT::TNode()("wire_type", "double");
+ case NUdf::EDataSlot::TzDate:
+ case NUdf::EDataSlot::TzDatetime:
+ case NUdf::EDataSlot::TzTimestamp:
+ return NYT::TNode()("wire_type", "string32");
+ case NUdf::EDataSlot::Decimal:
+ ythrow yexception() << "Decimal type without parameters.";
+ break;
+ }
+
+ ythrow yexception() << "Unsupported data type" << NUdf::GetDataTypeInfo(*slot).Name;
+ }
+
+ TMaybe<TType> LoadDataTypeParams(const TString& dataType, const TString& paramOne, const TString& /*paramTwo*/, ui32 /*level*/) {
+ const auto slot = NUdf::FindDataSlot(dataType);
+ if (!slot) {
+ ythrow yexception() << "Unsupported data type: " << dataType;
+ }
+
+ if (NUdf::EDataSlot::Decimal != slot) {
+ ythrow yexception() << "Unexpected parameters for type: " << dataType;
+ }
+
+ ui8 precision = 0;
+ if (!TryFromString(paramOne, precision) || !precision || precision > 35) {
+ ythrow yexception() << "Invalid decimal precision: " << paramOne;
+ }
+
+ if (NativeYTTypesFlags & NTCF_DECIMAL) {
+ if (precision < 10) {
+ return NYT::TNode()("wire_type", "int32");
+ } else if (precision < 19) {
+ return NYT::TNode()("wire_type", "int64");
+ } else {
+ return NYT::TNode()("wire_type", "int128");
+ }
+ }
+
+ return NYT::TNode()("wire_type", "string32");
+ }
+
+ TMaybe<TType> LoadResourceType(const TString& /*tag*/, ui32 /*level*/) {
+ ythrow yexception() << "Unsupported type: Resource";
+ }
+ TMaybe<TType> LoadTaggedType(TType baseType, const TString& /*tag*/, ui32 /*level*/) {
+ return baseType;
+ }
+ TMaybe<TType> LoadErrorType(ui32 /*row*/, ui32 /*column*/, const TString& /*file*/, const TString& msg, ui32 /*level*/) {
+ ythrow yexception() << msg;
+ }
+ TMaybe<TType> LoadStructType(const TVector<std::pair<TString, TType>>& members, ui32 level) {
+ if (!(NativeYTTypesFlags & NTCF_COMPLEX) && level > 0) {
+ return NYT::TNode()("wire_type", "yson32");
+ }
+ auto children = NYT::TNode::CreateList();
+
+ for (auto& item: members) {
+ NYT::TNode innerNode = item.second;
+ innerNode["name"] = item.first;
+ children.Add(std::move(innerNode));
+ }
+
+ return NYT::TNode()
+ ("wire_type", "tuple")
+ ("children", std::move(children));
+ }
+ TMaybe<TType> LoadListType(TType itemType, ui32 /*level*/) {
+ if (!(NativeYTTypesFlags & NTCF_COMPLEX)) {
+ return NYT::TNode()("wire_type", "yson32");
+ }
+ return NYT::TNode()
+ ("wire_type", "repeated_variant8")
+ ("children", NYT::TNode()
+ .Add(std::move(itemType))
+ );
+ }
+ TMaybe<TType> LoadStreamType(TType /*itemType*/, ui32 /*level*/) {
+ ythrow yexception() << "Unsupported type: Stream";
+ }
+ TMaybe<TType> LoadOptionalType(TType itemType, ui32 level) {
+ if (!(NativeYTTypesFlags & NTCF_COMPLEX) && level > 1) {
+ return NYT::TNode()("wire_type", "yson32");
+ }
+ return NYT::TNode()
+ ("wire_type", "variant8")
+ ("children", NYT::TNode()
+ .Add(NYT::TNode()("wire_type", "nothing"))
+ .Add(std::move(itemType))
+ );
+ }
+ TMaybe<TType> LoadTupleType(const TVector<TType>& elements, ui32 /*level*/) {
+ if (!(NativeYTTypesFlags & NTCF_COMPLEX)) {
+ return NYT::TNode()("wire_type", "yson32");
+ }
+
+ auto children = NYT::TNode::CreateList();
+ for (auto& inner: elements) {
+ children.Add(inner);
+ }
+
+ return NYT::TNode()
+ ("wire_type", "tuple")
+ ("children", std::move(children));
+ }
+ TMaybe<TType> LoadDictType(TType keyType, TType valType, ui32 /*level*/) {
+ if (!(NativeYTTypesFlags & NTCF_COMPLEX)) {
+ return NYT::TNode()("wire_type", "yson32");
+ }
+ return NYT::TNode()
+ ("wire_type", "repeated_variant8")
+ ("children", NYT::TNode()
+ .Add(NYT::TNode()
+ ("wire_type", "tuple")
+ ("children", NYT::TNode()
+ .Add(std::move(keyType))
+ .Add(std::move(valType))
+ )
+ )
+ );
+ }
+ TMaybe<TType> LoadCallableType(TType /*returnType*/, const TVector<TType>& /*argTypes*/, const TVector<TString>& /*argNames*/,
+ const TVector<ui64>& /*argFlags*/, size_t /*optionalCount*/, const TString& /*payload*/, ui32 /*level*/) {
+
+ ythrow yexception() << "Unsupported type: Callable";
+ }
+ TMaybe<TType> LoadVariantType(TType underlyingType, ui32 /*level*/) {
+ if (!(NativeYTTypesFlags & NTCF_COMPLEX)) {
+ return NYT::TNode()("wire_type", "yson32");
+ }
+ if (!underlyingType.IsMap() || !underlyingType.HasKey("children") || !underlyingType["children"].IsList()) {
+ ythrow yexception() << "Bad variant underlying type: " << NYT::NodeToYsonString(underlyingType);
+ }
+ auto altCount = underlyingType["children"].AsList().size();
+ underlyingType["wire_type"] = altCount < 256 ? "variant8" : "variant16";
+ return underlyingType;
+ }
+ void Error(const TString& info) {
+ ythrow yexception() << info;
+ }
+
+ ui64 NativeYTTypesFlags;
+};
+
+NYT::TNode ParseSkiffTypeFromYson(const NYT::TNode& node, ui64 nativeYTTypesFlags) {
+ TSkiffTypeLoader loader(nativeYTTypesFlags);
+ return DoLoadTypeFromYson(loader, node, 0).GetOrElse(NYT::TNode());
+}
+
} // namespace NCommon
} // namespace NYql
diff --git a/ydb/library/yql/providers/common/schema/skiff/yql_skiff_schema.h b/ydb/library/yql/providers/common/schema/skiff/yql_skiff_schema.h
index 2c9799057c..c5f8ec4522 100644
--- a/ydb/library/yql/providers/common/schema/skiff/yql_skiff_schema.h
+++ b/ydb/library/yql/providers/common/schema/skiff/yql_skiff_schema.h
@@ -5,7 +5,7 @@
namespace NYql {
namespace NCommon {
-NYT::TNode ParseSkiffTypeFromYson(const NYT::TNode& node, ui64 nativeYTTypesFlags);
-
+NYT::TNode ParseSkiffTypeFromYson(const NYT::TNode& node, ui64 nativeYTTypesFlags);
+
} // namespace NCommon
} // namespace NYql
diff --git a/ydb/library/yql/providers/common/schema/ya.make b/ydb/library/yql/providers/common/schema/ya.make
index f8334755d1..93cfc81008 100644
--- a/ydb/library/yql/providers/common/schema/ya.make
+++ b/ydb/library/yql/providers/common/schema/ya.make
@@ -1,14 +1,14 @@
-LIBRARY()
-
-OWNER(g:yql g:yql_ydb_core)
-
-SRCS(
- yql_schema_utils.cpp
-)
-
-PEERDIR(
- library/cpp/yson/node
+LIBRARY()
+
+OWNER(g:yql g:yql_ydb_core)
+
+SRCS(
+ yql_schema_utils.cpp
+)
+
+PEERDIR(
+ library/cpp/yson/node
ydb/library/yql/utils
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/providers/common/schema/yql_schema_utils.cpp b/ydb/library/yql/providers/common/schema/yql_schema_utils.cpp
index c8372260ac..1e03d6d975 100644
--- a/ydb/library/yql/providers/common/schema/yql_schema_utils.cpp
+++ b/ydb/library/yql/providers/common/schema/yql_schema_utils.cpp
@@ -1,7 +1,7 @@
-#include "yql_schema_utils.h"
+#include "yql_schema_utils.h"
#include <ydb/library/yql/utils/yql_panic.h>
-
+
namespace NYql {
namespace NCommon {
@@ -29,106 +29,106 @@ TVector<TString> ExtractColumnOrderFromYsonStructType(const NYT::TNode& node) {
return columns;
}
-bool EqualsYsonTypesIgnoreStructOrder(const NYT::TNode& left, const NYT::TNode& right) {
- auto typeName = left[0].AsString();
- if (typeName != right[0].AsString()) {
- return false;
- }
- if (left.Size() != right.Size()) {
- return false;
- }
- if (typeName == "VoidType") {
- return true;
- } else if (typeName == "NullType") {
- return true;
- } else if (typeName == "UnitType") {
- return true;
- } else if (typeName == "GenericType") {
- return true;
- } else if (typeName == "EmptyListType") {
- return true;
- } else if (typeName == "EmptyDictType") {
- return true;
- } else if (typeName == "DataType") {
- return left == right;
- } else if (typeName == "ResourceType") {
- return left[1].AsString() == right[1].AsString();
- } else if (typeName == "TaggedType") {
- return left[1].AsString() == right[1].AsString()
- && EqualsYsonTypesIgnoreStructOrder(left[2], right[2]);
- } else if (typeName == "ErrorType") {
- return left[1].AsInt64() == right[1].AsInt64() && left[2].AsInt64() == right[2].AsInt64()
- && left[3].AsString() == right[3].AsString() && left[4].AsString() == right[4].AsString();
- } else if (typeName == "StructType") {
- if (left[1].Size() != right[1].Size()) {
- return false;
- }
- THashMap<TString, size_t> members;
- for (size_t i = 0; i < right[1].Size(); ++i) {
- members.emplace(right[1][i][0].AsString(), i);
- }
- for (auto& item : left[1].AsList()) {
- auto name = item[0].AsString();
- auto it = members.find(name);
- if (it == members.end()) {
- return false;
- }
- if (!EqualsYsonTypesIgnoreStructOrder(item[1], right[1][it->second][1])) {
- return false;
- }
- }
- return true;
- } else if (typeName == "ListType") {
- return EqualsYsonTypesIgnoreStructOrder(left[1], right[1]);
- } else if (typeName == "StreamType") {
- return EqualsYsonTypesIgnoreStructOrder(left[1], right[1]);
- } else if (typeName == "OptionalType") {
- return EqualsYsonTypesIgnoreStructOrder(left[1], right[1]);
- } else if (typeName == "TupleType") {
- for (size_t i = 0; i < left[1].Size(); ++i) {
- if (!EqualsYsonTypesIgnoreStructOrder(left[1][i], right[1][i])) {
- return false;
- }
- }
- return true;
- } else if (typeName == "DictType") {
- return EqualsYsonTypesIgnoreStructOrder(left[1], right[1])
- && EqualsYsonTypesIgnoreStructOrder(left[2], right[2]);
- } else if (typeName == "CallableType") {
- if (left[1].Size() != right[1].Size() || left[2].Size() != right[2].Size() || left[3].Size() != right[3].Size()) {
- return false;
- }
- if (left[1].Size() > 0) {
- if (left[1][0].AsUint64() != right[1][0].AsUint64()) {
- return false;
- }
- if (left[1].Size() > 1) {
- if (left[1][1].AsString() != right[1][1].AsString()) {
- return false;
- }
- }
- }
-
- if (!EqualsYsonTypesIgnoreStructOrder(left[2][0], right[2][0])) {
- return false;
- }
-
- for (size_t i = 0; i < left[3].Size(); ++i) {
- if (!EqualsYsonTypesIgnoreStructOrder(left[3][i][0], right[3][i][0])) {
- return false;
- }
- for (size_t a = 1; a < left[3][i].Size(); ++a) {
- if (left[3][i][a] != right[3][i][a]) {
- return false;
- }
- }
- }
- return true;
- } else if (typeName == "VariantType") {
- return EqualsYsonTypesIgnoreStructOrder(left[1], right[1]);
- }
- return false;
-}
-
+bool EqualsYsonTypesIgnoreStructOrder(const NYT::TNode& left, const NYT::TNode& right) {
+ auto typeName = left[0].AsString();
+ if (typeName != right[0].AsString()) {
+ return false;
+ }
+ if (left.Size() != right.Size()) {
+ return false;
+ }
+ if (typeName == "VoidType") {
+ return true;
+ } else if (typeName == "NullType") {
+ return true;
+ } else if (typeName == "UnitType") {
+ return true;
+ } else if (typeName == "GenericType") {
+ return true;
+ } else if (typeName == "EmptyListType") {
+ return true;
+ } else if (typeName == "EmptyDictType") {
+ return true;
+ } else if (typeName == "DataType") {
+ return left == right;
+ } else if (typeName == "ResourceType") {
+ return left[1].AsString() == right[1].AsString();
+ } else if (typeName == "TaggedType") {
+ return left[1].AsString() == right[1].AsString()
+ && EqualsYsonTypesIgnoreStructOrder(left[2], right[2]);
+ } else if (typeName == "ErrorType") {
+ return left[1].AsInt64() == right[1].AsInt64() && left[2].AsInt64() == right[2].AsInt64()
+ && left[3].AsString() == right[3].AsString() && left[4].AsString() == right[4].AsString();
+ } else if (typeName == "StructType") {
+ if (left[1].Size() != right[1].Size()) {
+ return false;
+ }
+ THashMap<TString, size_t> members;
+ for (size_t i = 0; i < right[1].Size(); ++i) {
+ members.emplace(right[1][i][0].AsString(), i);
+ }
+ for (auto& item : left[1].AsList()) {
+ auto name = item[0].AsString();
+ auto it = members.find(name);
+ if (it == members.end()) {
+ return false;
+ }
+ if (!EqualsYsonTypesIgnoreStructOrder(item[1], right[1][it->second][1])) {
+ return false;
+ }
+ }
+ return true;
+ } else if (typeName == "ListType") {
+ return EqualsYsonTypesIgnoreStructOrder(left[1], right[1]);
+ } else if (typeName == "StreamType") {
+ return EqualsYsonTypesIgnoreStructOrder(left[1], right[1]);
+ } else if (typeName == "OptionalType") {
+ return EqualsYsonTypesIgnoreStructOrder(left[1], right[1]);
+ } else if (typeName == "TupleType") {
+ for (size_t i = 0; i < left[1].Size(); ++i) {
+ if (!EqualsYsonTypesIgnoreStructOrder(left[1][i], right[1][i])) {
+ return false;
+ }
+ }
+ return true;
+ } else if (typeName == "DictType") {
+ return EqualsYsonTypesIgnoreStructOrder(left[1], right[1])
+ && EqualsYsonTypesIgnoreStructOrder(left[2], right[2]);
+ } else if (typeName == "CallableType") {
+ if (left[1].Size() != right[1].Size() || left[2].Size() != right[2].Size() || left[3].Size() != right[3].Size()) {
+ return false;
+ }
+ if (left[1].Size() > 0) {
+ if (left[1][0].AsUint64() != right[1][0].AsUint64()) {
+ return false;
+ }
+ if (left[1].Size() > 1) {
+ if (left[1][1].AsString() != right[1][1].AsString()) {
+ return false;
+ }
+ }
+ }
+
+ if (!EqualsYsonTypesIgnoreStructOrder(left[2][0], right[2][0])) {
+ return false;
+ }
+
+ for (size_t i = 0; i < left[3].Size(); ++i) {
+ if (!EqualsYsonTypesIgnoreStructOrder(left[3][i][0], right[3][i][0])) {
+ return false;
+ }
+ for (size_t a = 1; a < left[3][i].Size(); ++a) {
+ if (left[3][i][a] != right[3][i][a]) {
+ return false;
+ }
+ }
+ }
+ return true;
+ } else if (typeName == "VariantType") {
+ return EqualsYsonTypesIgnoreStructOrder(left[1], right[1]);
+ }
+ return false;
+}
+
} // namespace NCommon
} // namespace NYql
diff --git a/ydb/library/yql/providers/common/schema/yql_schema_utils.h b/ydb/library/yql/providers/common/schema/yql_schema_utils.h
index 45061285a8..3d4cbc6f98 100644
--- a/ydb/library/yql/providers/common/schema/yql_schema_utils.h
+++ b/ydb/library/yql/providers/common/schema/yql_schema_utils.h
@@ -2,14 +2,14 @@
#include <library/cpp/yson/node/node.h>
-#include <util/generic/string.h>
-#include <util/generic/vector.h>
-
+#include <util/generic/string.h>
+#include <util/generic/vector.h>
+
namespace NYql {
namespace NCommon {
TVector<TString> ExtractColumnOrderFromYsonStructType(const NYT::TNode& node);
-bool EqualsYsonTypesIgnoreStructOrder(const NYT::TNode& left, const NYT::TNode& right);
-
+bool EqualsYsonTypesIgnoreStructOrder(const NYT::TNode& left, const NYT::TNode& right);
+
} // namespace NCommon
} // namespace NYql
diff --git a/ydb/library/yql/providers/common/transform/ya.make b/ydb/library/yql/providers/common/transform/ya.make
index 6a3b2475de..e979f8a210 100644
--- a/ydb/library/yql/providers/common/transform/ya.make
+++ b/ydb/library/yql/providers/common/transform/ya.make
@@ -1,23 +1,23 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(
g:yql
g:yql_ydb_core
)
-
-SRCS(
- yql_exec.cpp
- yql_lazy_init.cpp
- yql_optimize.cpp
- yql_visit.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ yql_exec.cpp
+ yql_lazy_init.cpp
+ yql_optimize.cpp
+ yql_visit.cpp
+)
+
+PEERDIR(
ydb/library/yql/ast
ydb/library/yql/utils
ydb/library/yql/utils/log
ydb/library/yql/core
ydb/library/yql/core/expr_nodes
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/providers/common/transform/yql_exec.cpp b/ydb/library/yql/providers/common/transform/yql_exec.cpp
index f771ae2b18..8269a60aef 100644
--- a/ydb/library/yql/providers/common/transform/yql_exec.cpp
+++ b/ydb/library/yql/providers/common/transform/yql_exec.cpp
@@ -1,101 +1,101 @@
-#include "yql_exec.h"
-
+#include "yql_exec.h"
+
#include <ydb/library/yql/core/yql_execution.h>
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/yql_panic.h>
-
-#include <util/string/builder.h>
-#include <util/generic/vector.h>
-#include <util/generic/string.h>
-
-
-namespace NYql {
-
-using namespace NNodes;
-
-TExecTransformerBase::TStatusCallbackPair TExecTransformerBase::CallbackTransform(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- YQL_ENSURE(input->Type() == TExprNode::Callable);
- output = input;
- if (auto handlerInfo = Handlers.FindPtr(input->Content())) {
- auto status = (handlerInfo->Prerequisite)(input);
- if (status.Level != TStatus::Ok) {
- return SyncStatus(status);
- }
- TString uniqId = TStringBuilder() << '#' << input->UniqueId();
- YQL_LOG_CTX_SCOPE(uniqId);
-
- return (handlerInfo->Handler)(input, output, ctx);
- }
-
+
+#include <util/string/builder.h>
+#include <util/generic/vector.h>
+#include <util/generic/string.h>
+
+
+namespace NYql {
+
+using namespace NNodes;
+
+TExecTransformerBase::TStatusCallbackPair TExecTransformerBase::CallbackTransform(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ YQL_ENSURE(input->Type() == TExprNode::Callable);
+ output = input;
+ if (auto handlerInfo = Handlers.FindPtr(input->Content())) {
+ auto status = (handlerInfo->Prerequisite)(input);
+ if (status.Level != TStatus::Ok) {
+ return SyncStatus(status);
+ }
+ TString uniqId = TStringBuilder() << '#' << input->UniqueId();
+ YQL_LOG_CTX_SCOPE(uniqId);
+
+ return (handlerInfo->Handler)(input, output, ctx);
+ }
+
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Don't know how to execute node: " << input->Content()));
- return SyncError();
-}
-
-void TExecTransformerBase::AddHandler(std::initializer_list<TStringBuf> names, TPrerequisite prerequisite, THandler handler) {
- THandlerInfo info;
- info.Handler = std::move(handler);
- info.Prerequisite = std::move(prerequisite);
- for (auto name: names) {
- YQL_ENSURE(Handlers.emplace(name, info).second, "Duplicate execution handler for " << name);
- }
-}
-
-TExecTransformerBase::THandler TExecTransformerBase::Pass() {
- return [] (const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
- return ExecPass(input, ctx);
- };
-}
-
-TExecTransformerBase::TStatusCallbackPair TExecTransformerBase::ExecPass(const TExprNode::TPtr& input, TExprContext& ctx) {
+ return SyncError();
+}
+
+void TExecTransformerBase::AddHandler(std::initializer_list<TStringBuf> names, TPrerequisite prerequisite, THandler handler) {
+ THandlerInfo info;
+ info.Handler = std::move(handler);
+ info.Prerequisite = std::move(prerequisite);
+ for (auto name: names) {
+ YQL_ENSURE(Handlers.emplace(name, info).second, "Duplicate execution handler for " << name);
+ }
+}
+
+TExecTransformerBase::THandler TExecTransformerBase::Pass() {
+ return [] (const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
+ return ExecPass(input, ctx);
+ };
+}
+
+TExecTransformerBase::TStatusCallbackPair TExecTransformerBase::ExecPass(const TExprNode::TPtr& input, TExprContext& ctx) {
input->SetState(TExprNode::EState::ExecutionComplete);
- input->SetResult(ctx.NewWorld(input->Pos()));
- return SyncOk();
-}
-
-TExecTransformerBase::TPrerequisite TExecTransformerBase::RequireAll() {
- return [] (const TExprNode::TPtr& input) {
- TStatus combinedStatus = TStatus::Ok;
- for (size_t i = 0; i < input->ChildrenSize(); ++i) {
- combinedStatus = combinedStatus.Combine(RequireChild(*input, (ui32)i));
- }
- return combinedStatus;
- };
-}
-
-TExecTransformerBase::TPrerequisite TExecTransformerBase::RequireNone() {
- return [] (const TExprNode::TPtr& /*input*/) {
- return TStatus::Ok;
- };
-}
-
-TExecTransformerBase::TPrerequisite TExecTransformerBase::RequireFirst() {
- return [] (const TExprNode::TPtr& input) {
- return RequireChild(*input, 0);
- };
-}
-
-TExecTransformerBase::TPrerequisite TExecTransformerBase::RequireAllOf(std::initializer_list<size_t> children) {
+ input->SetResult(ctx.NewWorld(input->Pos()));
+ return SyncOk();
+}
+
+TExecTransformerBase::TPrerequisite TExecTransformerBase::RequireAll() {
+ return [] (const TExprNode::TPtr& input) {
+ TStatus combinedStatus = TStatus::Ok;
+ for (size_t i = 0; i < input->ChildrenSize(); ++i) {
+ combinedStatus = combinedStatus.Combine(RequireChild(*input, (ui32)i));
+ }
+ return combinedStatus;
+ };
+}
+
+TExecTransformerBase::TPrerequisite TExecTransformerBase::RequireNone() {
+ return [] (const TExprNode::TPtr& /*input*/) {
+ return TStatus::Ok;
+ };
+}
+
+TExecTransformerBase::TPrerequisite TExecTransformerBase::RequireFirst() {
+ return [] (const TExprNode::TPtr& input) {
+ return RequireChild(*input, 0);
+ };
+}
+
+TExecTransformerBase::TPrerequisite TExecTransformerBase::RequireAllOf(std::initializer_list<size_t> children) {
return [required = TVector<size_t>(children)] (const TExprNode::TPtr& input) {
- TStatus combinedStatus = TStatus::Ok;
- for (size_t i: required) {
- YQL_ENSURE(i < input->ChildrenSize());
- combinedStatus = combinedStatus.Combine(RequireChild(*input, (ui32)i));
- }
- return combinedStatus;
- };
-}
-
-TExecTransformerBase::TPrerequisite TExecTransformerBase::RequireSequenceOf(std::initializer_list<size_t> children) {
+ TStatus combinedStatus = TStatus::Ok;
+ for (size_t i: required) {
+ YQL_ENSURE(i < input->ChildrenSize());
+ combinedStatus = combinedStatus.Combine(RequireChild(*input, (ui32)i));
+ }
+ return combinedStatus;
+ };
+}
+
+TExecTransformerBase::TPrerequisite TExecTransformerBase::RequireSequenceOf(std::initializer_list<size_t> children) {
return [required = TVector<size_t>(children)] (const TExprNode::TPtr& input) -> TStatus {
- for (size_t i: required) {
- YQL_ENSURE(i < input->ChildrenSize());
- auto status = RequireChild(*input, (ui32)i);
- if (status.Level != TStatus::Ok) {
- return status;
- }
- }
- return TStatus::Ok;
- };
-}
-
-}
+ for (size_t i: required) {
+ YQL_ENSURE(i < input->ChildrenSize());
+ auto status = RequireChild(*input, (ui32)i);
+ if (status.Level != TStatus::Ok) {
+ return status;
+ }
+ }
+ return TStatus::Ok;
+ };
+}
+
+}
diff --git a/ydb/library/yql/providers/common/transform/yql_exec.h b/ydb/library/yql/providers/common/transform/yql_exec.h
index 0f134e5688..a6f25716ef 100644
--- a/ydb/library/yql/providers/common/transform/yql_exec.h
+++ b/ydb/library/yql/providers/common/transform/yql_exec.h
@@ -3,68 +3,68 @@
#include <ydb/library/yql/core/yql_graph_transformer.h>
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/ast/yql_expr.h>
-
-#include <util/generic/strbuf.h>
-#include <util/generic/hash.h>
-
-#include <functional>
-#include <initializer_list>
-
+
+#include <util/generic/strbuf.h>
+#include <util/generic/hash.h>
+
+#include <functional>
+#include <initializer_list>
+
namespace NYql {
-class TExecTransformerBase : public TAsyncCallbackTransformer<TExecTransformerBase> {
-public:
- using TStatusCallbackPair = std::pair<TStatus, TAsyncTransformCallbackFuture>;
-
- using TPrerequisite = std::function<TStatus(const TExprNode::TPtr&)>;
- using THandler = std::function<TStatusCallbackPair(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)>;
-
+class TExecTransformerBase : public TAsyncCallbackTransformer<TExecTransformerBase> {
+public:
+ using TStatusCallbackPair = std::pair<TStatus, TAsyncTransformCallbackFuture>;
+
+ using TPrerequisite = std::function<TStatus(const TExprNode::TPtr&)>;
+ using THandler = std::function<TStatusCallbackPair(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)>;
+
TStatusCallbackPair CallbackTransform(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx);
-
- bool CanExec(const TExprNode& node) const {
+
+ bool CanExec(const TExprNode& node) const {
return Handlers.contains(node.Content());
- }
-
-protected:
- void AddHandler(std::initializer_list<TStringBuf> names, TPrerequisite prerequisite, THandler handler);
-
- static TPrerequisite RequireAll();
- static TPrerequisite RequireNone();
- static TPrerequisite RequireFirst();
- static TPrerequisite RequireAllOf(std::initializer_list<size_t> children);
- static TPrerequisite RequireSequenceOf(std::initializer_list<size_t> children);
-
- template <class TDerived>
- TPrerequisite Require(TStatus(TDerived::* prerequisite)(const TExprNode::TPtr&)) {
- return [this, prerequisite] (const TExprNode::TPtr& input) {
- return (static_cast<TDerived*>(this)->*prerequisite)(input);
- };
- }
-
-
- static TStatusCallbackPair ExecPass(const TExprNode::TPtr& input, TExprContext& ctx);
- static THandler Pass();
-
- template <class TDerived>
- THandler Hndl(TStatusCallbackPair(TDerived::* handler)(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)) {
- return [this, handler] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- return (static_cast<TDerived*>(this)->*handler)(input, output, ctx);
- };
- }
-
- template <class TDerived>
- THandler Hndl(TStatusCallbackPair(TDerived::* handler)(const TExprNode::TPtr&, TExprContext&)) {
- return [this, handler] (const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
- return (static_cast<TDerived*>(this)->*handler)(input, ctx);
- };
- }
-
-protected:
- struct THandlerInfo {
- TPrerequisite Prerequisite;
- THandler Handler;
- };
+ }
+
+protected:
+ void AddHandler(std::initializer_list<TStringBuf> names, TPrerequisite prerequisite, THandler handler);
+
+ static TPrerequisite RequireAll();
+ static TPrerequisite RequireNone();
+ static TPrerequisite RequireFirst();
+ static TPrerequisite RequireAllOf(std::initializer_list<size_t> children);
+ static TPrerequisite RequireSequenceOf(std::initializer_list<size_t> children);
+
+ template <class TDerived>
+ TPrerequisite Require(TStatus(TDerived::* prerequisite)(const TExprNode::TPtr&)) {
+ return [this, prerequisite] (const TExprNode::TPtr& input) {
+ return (static_cast<TDerived*>(this)->*prerequisite)(input);
+ };
+ }
+
+
+ static TStatusCallbackPair ExecPass(const TExprNode::TPtr& input, TExprContext& ctx);
+ static THandler Pass();
+
+ template <class TDerived>
+ THandler Hndl(TStatusCallbackPair(TDerived::* handler)(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)) {
+ return [this, handler] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ return (static_cast<TDerived*>(this)->*handler)(input, output, ctx);
+ };
+ }
+
+ template <class TDerived>
+ THandler Hndl(TStatusCallbackPair(TDerived::* handler)(const TExprNode::TPtr&, TExprContext&)) {
+ return [this, handler] (const TExprNode::TPtr& input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
+ return (static_cast<TDerived*>(this)->*handler)(input, ctx);
+ };
+ }
+
+protected:
+ struct THandlerInfo {
+ TPrerequisite Prerequisite;
+ THandler Handler;
+ };
THashMap<TStringBuf, THandlerInfo> Handlers;
-};
-
-} // NYql
+};
+
+} // NYql
diff --git a/ydb/library/yql/providers/common/transform/yql_lazy_init.cpp b/ydb/library/yql/providers/common/transform/yql_lazy_init.cpp
index 16ad10f2c5..4d66793639 100644
--- a/ydb/library/yql/providers/common/transform/yql_lazy_init.cpp
+++ b/ydb/library/yql/providers/common/transform/yql_lazy_init.cpp
@@ -1 +1 @@
-#include "yql_lazy_init.h"
+#include "yql_lazy_init.h"
diff --git a/ydb/library/yql/providers/common/transform/yql_lazy_init.h b/ydb/library/yql/providers/common/transform/yql_lazy_init.h
index fa2408e9ae..200fd913ae 100644
--- a/ydb/library/yql/providers/common/transform/yql_lazy_init.h
+++ b/ydb/library/yql/providers/common/transform/yql_lazy_init.h
@@ -1,35 +1,35 @@
#pragma once
-#include <util/generic/ptr.h>
-
-#include <functional>
-#include <utility>
-
+#include <util/generic/ptr.h>
+
+#include <functional>
+#include <utility>
+
namespace NYql {
-template <class T>
-class TLazyInitHolder
- : public TPointerBase<TLazyInitHolder<T>, T>
-{
-public:
- using TFactory = std::function<THolder<T>()>;
-
- TLazyInitHolder(TFactory&& factory)
- : Factory(std::move(factory))
- {
- }
-
+template <class T>
+class TLazyInitHolder
+ : public TPointerBase<TLazyInitHolder<T>, T>
+{
+public:
+ using TFactory = std::function<THolder<T>()>;
+
+ TLazyInitHolder(TFactory&& factory)
+ : Factory(std::move(factory))
+ {
+ }
+
T* Get() const noexcept {
- if (!Value) {
- Value = Factory();
- }
- return Value.Get();
- }
-
-private:
- TFactory Factory;
- mutable THolder<T> Value;
-
-};
-
-} // NYql
+ if (!Value) {
+ Value = Factory();
+ }
+ return Value.Get();
+ }
+
+private:
+ TFactory Factory;
+ mutable THolder<T> Value;
+
+};
+
+} // NYql
diff --git a/ydb/library/yql/providers/common/transform/yql_optimize.cpp b/ydb/library/yql/providers/common/transform/yql_optimize.cpp
index ef29baa8ff..127a41a003 100644
--- a/ydb/library/yql/providers/common/transform/yql_optimize.cpp
+++ b/ydb/library/yql/providers/common/transform/yql_optimize.cpp
@@ -1,209 +1,209 @@
-#include "yql_optimize.h"
-
+#include "yql_optimize.h"
+
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/yql_panic.h>
-
-#include <util/generic/hash_set.h>
-#include <util/generic/yexception.h>
-
-
-namespace NYql {
-
-using namespace NNodes;
-
-class TOptimizeTransformerBase::TIgnoreOptimizationContext: public IOptimizationContext {
-public:
- TIgnoreOptimizationContext(TOptimizeTransformerBase::TGetParents getParents)
- : GetParents(std::move(getParents))
- {
- }
- virtual ~TIgnoreOptimizationContext() = default;
- void RemapNode(const TExprNode& src, const TExprNode::TPtr&) final {
- const TParentsMap* parentsMap = GetParents();
- auto parentsIt = parentsMap->find(&src);
- YQL_ENSURE(parentsIt != parentsMap->cend());
- YQL_ENSURE(parentsIt->second.size() == 1, "Bad usage of local optimizer. Try to switch to global mode");
- }
-private:
- TOptimizeTransformerBase::TGetParents GetParents;
-};
-
-class TOptimizeTransformerBase::TRemapOptimizationContext: public IOptimizationContext {
-public:
- TRemapOptimizationContext(TNodeOnNodeOwnedMap& remaps)
- : Remaps_(remaps)
- {
- }
- virtual ~TRemapOptimizationContext() = default;
- void RemapNode(const TExprNode& fromNode, const TExprNode::TPtr& toNode) final {
- YQL_ENSURE(Remaps_.emplace(&fromNode, toNode).second, "Duplicate remap of the same node");
- }
- void SetError() {
- HasError_ = true;
- }
- bool CanContinue() const {
- return Remaps_.empty() && !HasError_;
- }
- bool HasError() const {
- return HasError_;
- }
-private:
- TNodeOnNodeOwnedMap& Remaps_;
- bool HasError_ = false;
-};
-
-
-TOptimizeTransformerBase::TOptimizeTransformerBase(TTypeAnnotationContext* types, NLog::EComponent logComponent, const TSet<TString>& disabledOpts)
+
+#include <util/generic/hash_set.h>
+#include <util/generic/yexception.h>
+
+
+namespace NYql {
+
+using namespace NNodes;
+
+class TOptimizeTransformerBase::TIgnoreOptimizationContext: public IOptimizationContext {
+public:
+ TIgnoreOptimizationContext(TOptimizeTransformerBase::TGetParents getParents)
+ : GetParents(std::move(getParents))
+ {
+ }
+ virtual ~TIgnoreOptimizationContext() = default;
+ void RemapNode(const TExprNode& src, const TExprNode::TPtr&) final {
+ const TParentsMap* parentsMap = GetParents();
+ auto parentsIt = parentsMap->find(&src);
+ YQL_ENSURE(parentsIt != parentsMap->cend());
+ YQL_ENSURE(parentsIt->second.size() == 1, "Bad usage of local optimizer. Try to switch to global mode");
+ }
+private:
+ TOptimizeTransformerBase::TGetParents GetParents;
+};
+
+class TOptimizeTransformerBase::TRemapOptimizationContext: public IOptimizationContext {
+public:
+ TRemapOptimizationContext(TNodeOnNodeOwnedMap& remaps)
+ : Remaps_(remaps)
+ {
+ }
+ virtual ~TRemapOptimizationContext() = default;
+ void RemapNode(const TExprNode& fromNode, const TExprNode::TPtr& toNode) final {
+ YQL_ENSURE(Remaps_.emplace(&fromNode, toNode).second, "Duplicate remap of the same node");
+ }
+ void SetError() {
+ HasError_ = true;
+ }
+ bool CanContinue() const {
+ return Remaps_.empty() && !HasError_;
+ }
+ bool HasError() const {
+ return HasError_;
+ }
+private:
+ TNodeOnNodeOwnedMap& Remaps_;
+ bool HasError_ = false;
+};
+
+
+TOptimizeTransformerBase::TOptimizeTransformerBase(TTypeAnnotationContext* types, NLog::EComponent logComponent, const TSet<TString>& disabledOpts)
: Types(types)
- , LogComponent(logComponent)
- , DisabledOpts(disabledOpts)
-{
-}
-
-IGraphTransformer::TStatus TOptimizeTransformerBase::DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) {
- TOptimizeExprSettings settings(Types);
- IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok;
-
- output = input;
- for (auto& step: Steps) {
- TParentsMap parentsMap;
- bool parentsMapInit = false;
- TGetParents getParents = [&input, &parentsMap, &parentsMapInit] () {
- if (!parentsMapInit) {
- GatherParents(*input, parentsMap);
- parentsMapInit = true;
- }
- return &parentsMap;
- };
- if (step.Global) {
- TNodeOnNodeOwnedMap remaps;
- auto optCtx = TRemapOptimizationContext{remaps};
- VisitExpr(output,
- [&optCtx, &step](const TExprNode::TPtr& node) {
- return optCtx.CanContinue() && !node->StartsExecution() && !step.ProcessedNodes.contains(node->UniqueId());
- },
- [this, &step, &getParents, &ctx, &optCtx](const TExprNode::TPtr& node) -> bool {
- if (optCtx.CanContinue() && !node->StartsExecution() && !step.ProcessedNodes.contains(node->UniqueId())) {
- for (auto& opt: step.Optimizers) {
- if (opt.Filter(node.Get())) {
- try {
- auto ret = opt.Handler(NNodes::TExprBase(node), ctx, optCtx, getParents);
- if (!ret) {
- YQL_CVLOG(NLog::ELevel::ERROR, LogComponent) << "Error applying " << opt.OptName;
- optCtx.SetError();
- } else if (auto retNode = ret.Cast(); retNode.Ptr() != node) {
- YQL_CVLOG(NLog::ELevel::INFO, LogComponent) << opt.OptName;
- optCtx.RemapNode(*node, retNode.Ptr());
- }
- } catch (...) {
- YQL_CVLOG(NLog::ELevel::ERROR, LogComponent) << "Error applying " << opt.OptName << ": " << CurrentExceptionMessage();
- throw;
- }
- }
- if (!optCtx.CanContinue()) {
- break;
- }
- }
- if (optCtx.CanContinue()) {
- step.ProcessedNodes.insert(node->UniqueId());
- }
- }
- return true;
- }
- );
-
- if (optCtx.HasError()) {
- status = IGraphTransformer::TStatus::Error;
- } else if (!remaps.empty()) {
- settings.ProcessedNodes = nullptr;
- status = RemapExpr(output, output, remaps, ctx, settings);
- }
- } else {
- settings.ProcessedNodes = &step.ProcessedNodes;
- status = OptimizeExpr(output, output, [this, &step, &getParents](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
- TIgnoreOptimizationContext ignoreOptCtx(getParents);
- for (auto& opt: step.Optimizers) {
- if (opt.Filter(node.Get())) {
- try {
- auto ret = opt.Handler(NNodes::TExprBase(node), ctx, ignoreOptCtx, getParents);
- if (!ret) {
- YQL_CVLOG(NLog::ELevel::ERROR, LogComponent) << "Error applying " << opt.OptName;
- return {};
- }
- auto retNode = ret.Cast();
- if (retNode.Ptr() != node) {
- YQL_CVLOG(NLog::ELevel::INFO, LogComponent) << opt.OptName;
- return retNode.Ptr();
- }
- } catch (...) {
- YQL_CVLOG(NLog::ELevel::ERROR, LogComponent) << "Error applying " << opt.OptName << ": " << CurrentExceptionMessage();
- throw;
- }
- }
- }
- return node;
- }, ctx, settings);
- }
-
- if (status.Level != IGraphTransformer::TStatus::Ok) {
- return status;
- }
- }
-
- return status;
-}
-
-void TOptimizeTransformerBase::Rewind() {
- for (auto& step: Steps) {
- step.ProcessedNodes.clear();
- }
-}
-
-TOptimizeTransformerBase::TFilter TOptimizeTransformerBase::Any() {
- return [] (const TExprNode* node) {
- Y_UNUSED(node);
- return true;
- };
-}
-
-TOptimizeTransformerBase::TFilter TOptimizeTransformerBase::Names(std::initializer_list<TStringBuf> names) {
+ , LogComponent(logComponent)
+ , DisabledOpts(disabledOpts)
+{
+}
+
+IGraphTransformer::TStatus TOptimizeTransformerBase::DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) {
+ TOptimizeExprSettings settings(Types);
+ IGraphTransformer::TStatus status = IGraphTransformer::TStatus::Ok;
+
+ output = input;
+ for (auto& step: Steps) {
+ TParentsMap parentsMap;
+ bool parentsMapInit = false;
+ TGetParents getParents = [&input, &parentsMap, &parentsMapInit] () {
+ if (!parentsMapInit) {
+ GatherParents(*input, parentsMap);
+ parentsMapInit = true;
+ }
+ return &parentsMap;
+ };
+ if (step.Global) {
+ TNodeOnNodeOwnedMap remaps;
+ auto optCtx = TRemapOptimizationContext{remaps};
+ VisitExpr(output,
+ [&optCtx, &step](const TExprNode::TPtr& node) {
+ return optCtx.CanContinue() && !node->StartsExecution() && !step.ProcessedNodes.contains(node->UniqueId());
+ },
+ [this, &step, &getParents, &ctx, &optCtx](const TExprNode::TPtr& node) -> bool {
+ if (optCtx.CanContinue() && !node->StartsExecution() && !step.ProcessedNodes.contains(node->UniqueId())) {
+ for (auto& opt: step.Optimizers) {
+ if (opt.Filter(node.Get())) {
+ try {
+ auto ret = opt.Handler(NNodes::TExprBase(node), ctx, optCtx, getParents);
+ if (!ret) {
+ YQL_CVLOG(NLog::ELevel::ERROR, LogComponent) << "Error applying " << opt.OptName;
+ optCtx.SetError();
+ } else if (auto retNode = ret.Cast(); retNode.Ptr() != node) {
+ YQL_CVLOG(NLog::ELevel::INFO, LogComponent) << opt.OptName;
+ optCtx.RemapNode(*node, retNode.Ptr());
+ }
+ } catch (...) {
+ YQL_CVLOG(NLog::ELevel::ERROR, LogComponent) << "Error applying " << opt.OptName << ": " << CurrentExceptionMessage();
+ throw;
+ }
+ }
+ if (!optCtx.CanContinue()) {
+ break;
+ }
+ }
+ if (optCtx.CanContinue()) {
+ step.ProcessedNodes.insert(node->UniqueId());
+ }
+ }
+ return true;
+ }
+ );
+
+ if (optCtx.HasError()) {
+ status = IGraphTransformer::TStatus::Error;
+ } else if (!remaps.empty()) {
+ settings.ProcessedNodes = nullptr;
+ status = RemapExpr(output, output, remaps, ctx, settings);
+ }
+ } else {
+ settings.ProcessedNodes = &step.ProcessedNodes;
+ status = OptimizeExpr(output, output, [this, &step, &getParents](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
+ TIgnoreOptimizationContext ignoreOptCtx(getParents);
+ for (auto& opt: step.Optimizers) {
+ if (opt.Filter(node.Get())) {
+ try {
+ auto ret = opt.Handler(NNodes::TExprBase(node), ctx, ignoreOptCtx, getParents);
+ if (!ret) {
+ YQL_CVLOG(NLog::ELevel::ERROR, LogComponent) << "Error applying " << opt.OptName;
+ return {};
+ }
+ auto retNode = ret.Cast();
+ if (retNode.Ptr() != node) {
+ YQL_CVLOG(NLog::ELevel::INFO, LogComponent) << opt.OptName;
+ return retNode.Ptr();
+ }
+ } catch (...) {
+ YQL_CVLOG(NLog::ELevel::ERROR, LogComponent) << "Error applying " << opt.OptName << ": " << CurrentExceptionMessage();
+ throw;
+ }
+ }
+ }
+ return node;
+ }, ctx, settings);
+ }
+
+ if (status.Level != IGraphTransformer::TStatus::Ok) {
+ return status;
+ }
+ }
+
+ return status;
+}
+
+void TOptimizeTransformerBase::Rewind() {
+ for (auto& step: Steps) {
+ step.ProcessedNodes.clear();
+ }
+}
+
+TOptimizeTransformerBase::TFilter TOptimizeTransformerBase::Any() {
+ return [] (const TExprNode* node) {
+ Y_UNUSED(node);
+ return true;
+ };
+}
+
+TOptimizeTransformerBase::TFilter TOptimizeTransformerBase::Names(std::initializer_list<TStringBuf> names) {
return [filter = THashSet<TStringBuf>(names)] (const TExprNode* node) {
- return node->IsCallable(filter);
- };
-}
-
-TOptimizeTransformerBase::TFilter TOptimizeTransformerBase::Or(std::initializer_list<TOptimizeTransformerBase::TFilter> filters) {
- return [orFilters = TVector<TFilter>(filters)] (const TExprNode* node) {
- for (auto& f: orFilters) {
- if (f(node)) {
- return true;
- }
- }
- return false;
- };
-}
-
-void TOptimizeTransformerBase::AddHandler(size_t step, TFilter filter, TStringBuf optName, THandler handler) {
+ return node->IsCallable(filter);
+ };
+}
+
+TOptimizeTransformerBase::TFilter TOptimizeTransformerBase::Or(std::initializer_list<TOptimizeTransformerBase::TFilter> filters) {
+ return [orFilters = TVector<TFilter>(filters)] (const TExprNode* node) {
+ for (auto& f: orFilters) {
+ if (f(node)) {
+ return true;
+ }
+ }
+ return false;
+ };
+}
+
+void TOptimizeTransformerBase::AddHandler(size_t step, TFilter filter, TStringBuf optName, THandler handler) {
if (DisabledOpts.contains(optName)) {
- return;
- }
-
- if (step >= Steps.size()) {
- Steps.resize(step + 1);
- }
-
- TOptInfo opt;
- opt.OptName = optName;
- opt.Filter = filter;
- opt.Handler = handler;
- Steps[step].Optimizers.push_back(std::move(opt));
-}
-
-void TOptimizeTransformerBase::SetGlobal(size_t step) {
- if (step >= Steps.size()) {
- Steps.resize(step + 1);
- }
- Steps[step].Global = true;
-}
-
-}
+ return;
+ }
+
+ if (step >= Steps.size()) {
+ Steps.resize(step + 1);
+ }
+
+ TOptInfo opt;
+ opt.OptName = optName;
+ opt.Filter = filter;
+ opt.Handler = handler;
+ Steps[step].Optimizers.push_back(std::move(opt));
+}
+
+void TOptimizeTransformerBase::SetGlobal(size_t step) {
+ if (step >= Steps.size()) {
+ Steps.resize(step + 1);
+ }
+ Steps[step].Global = true;
+}
+
+}
diff --git a/ydb/library/yql/providers/common/transform/yql_optimize.h b/ydb/library/yql/providers/common/transform/yql_optimize.h
index d4d65b84f1..05ccd34879 100644
--- a/ydb/library/yql/providers/common/transform/yql_optimize.h
+++ b/ydb/library/yql/providers/common/transform/yql_optimize.h
@@ -5,110 +5,110 @@
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/utils/log/log_component.h>
-
-#include <util/generic/vector.h>
-#include <util/generic/strbuf.h>
-#include <util/generic/ptr.h>
-#include <util/generic/set.h>
-#include <util/generic/string.h>
-
-#include <functional>
-#include <initializer_list>
-
+
+#include <util/generic/vector.h>
+#include <util/generic/strbuf.h>
+#include <util/generic/ptr.h>
+#include <util/generic/set.h>
+#include <util/generic/string.h>
+
+#include <functional>
+#include <initializer_list>
+
namespace NYql {
-class TOptimizeTransformerBase: public TSyncTransformerBase {
-public:
- using TGetParents = std::function<const TParentsMap*()>;
- using THandler = std::function<NNodes::TMaybeNode<NNodes::TExprBase>(NNodes::TExprBase, TExprContext&, IOptimizationContext&, const TGetParents&)>;
- using TFilter = std::function<bool(const TExprNode*)>;
-
- TOptimizeTransformerBase(TTypeAnnotationContext* types, NLog::EComponent logComponent, const TSet<TString>& disabledOpts);
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override;
- void Rewind() override;
-
-protected:
- class TIgnoreOptimizationContext;
- class TRemapOptimizationContext;
-
- static TFilter Any();
- static TFilter Names(std::initializer_list<TStringBuf> names);
- static TFilter Or(std::initializer_list<TFilter> filters);
-
- void AddHandler(size_t step, TFilter filter, TStringBuf optName, THandler handler);
- void SetGlobal(size_t step);
-
- template <class TDerived>
- THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, const TGetParents&)) {
- return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& /*optCtx*/, const TGetParents& parents) {
- return (static_cast<TDerived*>(this)->*handler)(node, ctx, parents);
- };
- }
-
- template <class TDerived>
- THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, const TGetParents&) const) const {
- return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& /*optCtx*/, const TGetParents& parents) {
- return (static_cast<const TDerived*>(this)->*handler)(node, ctx, parents);
- };
- }
-
- template <class TDerived>
- THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, IOptimizationContext&, const TGetParents&)) {
- return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& parents) {
- return (static_cast<TDerived*>(this)->*handler)(node, ctx, optCtx, parents);
- };
- }
-
- template <class TDerived>
- THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, IOptimizationContext&, const TGetParents&) const) const {
- return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& parents) {
- return (static_cast<const TDerived*>(this)->*handler)(node, ctx, optCtx, parents);
- };
- }
-
- template <class TDerived>
- THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, IOptimizationContext&)) {
- return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& /*parents*/) {
- return (static_cast<TDerived*>(this)->*handler)(node, ctx, optCtx);
- };
- }
-
- template <class TDerived>
- THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, IOptimizationContext&) const) const {
- return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& /*parents*/) {
- return (static_cast<const TDerived*>(this)->*handler)(node, ctx, optCtx);
- };
- }
-
- template <class TDerived>
- THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&)) {
- return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& /*optCtx*/, const TGetParents& /*parents*/) {
- return (static_cast<TDerived*>(this)->*handler)(node, ctx);
- };
- }
-
- template <class TDerived>
- THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&) const) const {
- return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& /*optCtx*/, const TGetParents& /*parents*/) {
- return (static_cast<const TDerived*>(this)->*handler)(node, ctx);
- };
- }
-
-protected:
- struct TOptInfo {
- TString OptName;
- TFilter Filter;
- THandler Handler;
- };
- struct TStep {
- TProcessedNodesSet ProcessedNodes;
+class TOptimizeTransformerBase: public TSyncTransformerBase {
+public:
+ using TGetParents = std::function<const TParentsMap*()>;
+ using THandler = std::function<NNodes::TMaybeNode<NNodes::TExprBase>(NNodes::TExprBase, TExprContext&, IOptimizationContext&, const TGetParents&)>;
+ using TFilter = std::function<bool(const TExprNode*)>;
+
+ TOptimizeTransformerBase(TTypeAnnotationContext* types, NLog::EComponent logComponent, const TSet<TString>& disabledOpts);
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) override;
+ void Rewind() override;
+
+protected:
+ class TIgnoreOptimizationContext;
+ class TRemapOptimizationContext;
+
+ static TFilter Any();
+ static TFilter Names(std::initializer_list<TStringBuf> names);
+ static TFilter Or(std::initializer_list<TFilter> filters);
+
+ void AddHandler(size_t step, TFilter filter, TStringBuf optName, THandler handler);
+ void SetGlobal(size_t step);
+
+ template <class TDerived>
+ THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, const TGetParents&)) {
+ return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& /*optCtx*/, const TGetParents& parents) {
+ return (static_cast<TDerived*>(this)->*handler)(node, ctx, parents);
+ };
+ }
+
+ template <class TDerived>
+ THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, const TGetParents&) const) const {
+ return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& /*optCtx*/, const TGetParents& parents) {
+ return (static_cast<const TDerived*>(this)->*handler)(node, ctx, parents);
+ };
+ }
+
+ template <class TDerived>
+ THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, IOptimizationContext&, const TGetParents&)) {
+ return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& parents) {
+ return (static_cast<TDerived*>(this)->*handler)(node, ctx, optCtx, parents);
+ };
+ }
+
+ template <class TDerived>
+ THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, IOptimizationContext&, const TGetParents&) const) const {
+ return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& parents) {
+ return (static_cast<const TDerived*>(this)->*handler)(node, ctx, optCtx, parents);
+ };
+ }
+
+ template <class TDerived>
+ THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, IOptimizationContext&)) {
+ return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& /*parents*/) {
+ return (static_cast<TDerived*>(this)->*handler)(node, ctx, optCtx);
+ };
+ }
+
+ template <class TDerived>
+ THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&, IOptimizationContext&) const) const {
+ return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& /*parents*/) {
+ return (static_cast<const TDerived*>(this)->*handler)(node, ctx, optCtx);
+ };
+ }
+
+ template <class TDerived>
+ THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&)) {
+ return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& /*optCtx*/, const TGetParents& /*parents*/) {
+ return (static_cast<TDerived*>(this)->*handler)(node, ctx);
+ };
+ }
+
+ template <class TDerived>
+ THandler Hndl(NNodes::TMaybeNode<NNodes::TExprBase>(TDerived::* handler)(NNodes::TExprBase, TExprContext&) const) const {
+ return [this, handler] (NNodes::TExprBase node, TExprContext& ctx, IOptimizationContext& /*optCtx*/, const TGetParents& /*parents*/) {
+ return (static_cast<const TDerived*>(this)->*handler)(node, ctx);
+ };
+ }
+
+protected:
+ struct TOptInfo {
+ TString OptName;
+ TFilter Filter;
+ THandler Handler;
+ };
+ struct TStep {
+ TProcessedNodesSet ProcessedNodes;
TVector<TOptInfo> Optimizers;
- bool Global = false;
- };
- TTypeAnnotationContext* Types;
- const NLog::EComponent LogComponent;
- TSet<TString> DisabledOpts;
+ bool Global = false;
+ };
+ TTypeAnnotationContext* Types;
+ const NLog::EComponent LogComponent;
+ TSet<TString> DisabledOpts;
TVector<TStep> Steps;
-};
-
-} // NYql
+};
+
+} // NYql
diff --git a/ydb/library/yql/providers/common/transform/yql_visit.cpp b/ydb/library/yql/providers/common/transform/yql_visit.cpp
index c6ec8a4a13..0cd7bd8237 100644
--- a/ydb/library/yql/providers/common/transform/yql_visit.cpp
+++ b/ydb/library/yql/providers/common/transform/yql_visit.cpp
@@ -1,29 +1,29 @@
-#include "yql_visit.h"
-
+#include "yql_visit.h"
+
#include <ydb/library/yql/utils/yql_panic.h>
-
-#include <util/string/builder.h>
-
-namespace NYql {
-
-void TVisitorTransformerBase::AddHandler(std::initializer_list<TStringBuf> names, THandler handler) {
- for (auto name: names) {
- YQL_ENSURE(Handlers.emplace(name, handler).second, "Duplicate handler for " << name);
- }
-}
-
-IGraphTransformer::TStatus TVisitorTransformerBase::DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) {
- YQL_ENSURE(input->Type() == TExprNode::Callable);
- output = input;
-
- if (auto handler = Handlers.FindPtr(input->Content())) {
- return (*handler)(input, output, ctx);
- }
- if (FailOnUnknown) {
+
+#include <util/string/builder.h>
+
+namespace NYql {
+
+void TVisitorTransformerBase::AddHandler(std::initializer_list<TStringBuf> names, THandler handler) {
+ for (auto name: names) {
+ YQL_ENSURE(Handlers.emplace(name, handler).second, "Duplicate handler for " << name);
+ }
+}
+
+IGraphTransformer::TStatus TVisitorTransformerBase::DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) {
+ YQL_ENSURE(input->Type() == TExprNode::Callable);
+ output = input;
+
+ if (auto handler = Handlers.FindPtr(input->Content())) {
+ return (*handler)(input, output, ctx);
+ }
+ if (FailOnUnknown) {
ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "Unsupported callable: " << input->Content()));
- return TStatus::Error;
- }
- return TStatus::Ok;
-}
-
-}
+ return TStatus::Error;
+ }
+ return TStatus::Ok;
+}
+
+}
diff --git a/ydb/library/yql/providers/common/transform/yql_visit.h b/ydb/library/yql/providers/common/transform/yql_visit.h
index 5e6212579e..e64d4f13c6 100644
--- a/ydb/library/yql/providers/common/transform/yql_visit.h
+++ b/ydb/library/yql/providers/common/transform/yql_visit.h
@@ -3,69 +3,69 @@
#include <ydb/library/yql/core/yql_graph_transformer.h>
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/ast/yql_expr.h>
-
-#include <util/generic/hash.h>
-#include <util/generic/strbuf.h>
-
-#include <functional>
-#include <initializer_list>
-
+
+#include <util/generic/hash.h>
+#include <util/generic/strbuf.h>
+
+#include <functional>
+#include <initializer_list>
+
namespace NYql {
-class TVisitorTransformerBase: public TSyncTransformerBase {
-public:
- using THandler = std::function<TStatus(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)>;
-
- TVisitorTransformerBase(bool failOnUnknown)
- : FailOnUnknown(failOnUnknown)
- {
- }
+class TVisitorTransformerBase: public TSyncTransformerBase {
+public:
+ using THandler = std::function<TStatus(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)>;
+ TVisitorTransformerBase(bool failOnUnknown)
+ : FailOnUnknown(failOnUnknown)
+ {
+ }
+
TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final;
-
- bool CanParse(const TExprNode& node) const {
+
+ bool CanParse(const TExprNode& node) const {
return Handlers.contains(node.Content());
- }
-
-protected:
- void AddHandler(std::initializer_list<TStringBuf> names, THandler handler);
-
- template <class TDerived>
- THandler Hndl(TStatus(TDerived::* handler)(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)) {
- return [this, handler] (TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) {
- return (static_cast<TDerived*>(this)->*handler)(input, output, ctx);
- };
- }
-
- template <class TDerived>
- THandler Hndl(TStatus(TDerived::* handler)(const TExprNode::TPtr&, TExprContext&)) {
- return [this, handler] (TExprNode::TPtr input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
- return (static_cast<TDerived*>(this)->*handler)(input, ctx);
- };
- }
-
- template <class TDerived>
- THandler Hndl(TStatus(TDerived::* handler)(NNodes::TExprBase, TExprContext&)) {
- return [this, handler] (TExprNode::TPtr input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
- return (static_cast<TDerived*>(this)->*handler)(NNodes::TExprBase(input), ctx);
- };
- }
-
- THandler Hndl(TStatus(*handler)(const TExprNode::TPtr&, TExprContext&)) {
- return [handler] (TExprNode::TPtr input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
- return handler(input, ctx);
- };
- }
-
- THandler Hndl(TStatus(*handler)(NNodes::TExprBase, TExprContext&)) {
- return [handler] (TExprNode::TPtr input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
- return handler(NNodes::TExprBase(input), ctx);
- };
- }
-
-protected:
- const bool FailOnUnknown;
+ }
+
+protected:
+ void AddHandler(std::initializer_list<TStringBuf> names, THandler handler);
+
+ template <class TDerived>
+ THandler Hndl(TStatus(TDerived::* handler)(const TExprNode::TPtr&, TExprNode::TPtr&, TExprContext&)) {
+ return [this, handler] (TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) {
+ return (static_cast<TDerived*>(this)->*handler)(input, output, ctx);
+ };
+ }
+
+ template <class TDerived>
+ THandler Hndl(TStatus(TDerived::* handler)(const TExprNode::TPtr&, TExprContext&)) {
+ return [this, handler] (TExprNode::TPtr input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
+ return (static_cast<TDerived*>(this)->*handler)(input, ctx);
+ };
+ }
+
+ template <class TDerived>
+ THandler Hndl(TStatus(TDerived::* handler)(NNodes::TExprBase, TExprContext&)) {
+ return [this, handler] (TExprNode::TPtr input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
+ return (static_cast<TDerived*>(this)->*handler)(NNodes::TExprBase(input), ctx);
+ };
+ }
+
+ THandler Hndl(TStatus(*handler)(const TExprNode::TPtr&, TExprContext&)) {
+ return [handler] (TExprNode::TPtr input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
+ return handler(input, ctx);
+ };
+ }
+
+ THandler Hndl(TStatus(*handler)(NNodes::TExprBase, TExprContext&)) {
+ return [handler] (TExprNode::TPtr input, TExprNode::TPtr& /*output*/, TExprContext& ctx) {
+ return handler(NNodes::TExprBase(input), ctx);
+ };
+ }
+
+protected:
+ const bool FailOnUnknown;
THashMap<TStringBuf, THandler> Handlers;
-};
-
-} // NYql
+};
+
+} // NYql
diff --git a/ydb/library/yql/providers/common/udf_resolve/ya.make b/ydb/library/yql/providers/common/udf_resolve/ya.make
index a9d49f1ef8..3c2f4eb59d 100644
--- a/ydb/library/yql/providers/common/udf_resolve/ya.make
+++ b/ydb/library/yql/providers/common/udf_resolve/ya.make
@@ -1,23 +1,23 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(
spuchin
g:yql
g:yql_ydb_core
)
-
-SRCS(
- yql_files_box.cpp
- yql_files_box.h
- yql_outproc_udf_resolver.cpp
- yql_outproc_udf_resolver.h
- yql_simple_udf_resolver.cpp
- yql_simple_udf_resolver.h
- yql_udf_resolver_with_index.cpp
- yql_udf_resolver_with_index.h
-)
-
-PEERDIR(
+
+SRCS(
+ yql_files_box.cpp
+ yql_files_box.h
+ yql_outproc_udf_resolver.cpp
+ yql_outproc_udf_resolver.h
+ yql_simple_udf_resolver.cpp
+ yql_simple_udf_resolver.h
+ yql_udf_resolver_with_index.cpp
+ yql_udf_resolver_with_index.h
+)
+
+PEERDIR(
library/cpp/protobuf/util
ydb/library/yql/core/file_storage
ydb/library/yql/minikql
@@ -28,8 +28,8 @@ PEERDIR(
ydb/library/yql/providers/common/mkql
ydb/library/yql/providers/common/proto
ydb/library/yql/providers/common/schema/expr
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_files_box.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_files_box.cpp
index 95fe68f080..9b69010aba 100644
--- a/ydb/library/yql/providers/common/udf_resolve/yql_files_box.cpp
+++ b/ydb/library/yql/providers/common/udf_resolve/yql_files_box.cpp
@@ -2,11 +2,11 @@
#include <ydb/library/yql/core/file_storage/storage.h>
#include <ydb/library/yql/utils/log/log.h>
-
+
#include <util/system/fs.h>
-#include <util/system/error.h>
-#include <util/system/sysstat.h>
-#include <util/folder/dirut.h>
+#include <util/system/error.h>
+#include <util/system/sysstat.h>
+#include <util/folder/dirut.h>
namespace NYql {
namespace NCommon {
@@ -34,21 +34,21 @@ TString TFilesBox::MakeLinkFrom(const TString& source, const TString& filename)
TFsPath sourcePath(source);
TString sourceAbsolutePath = sourcePath.IsAbsolute() ? source : (TFsPath::Cwd() / sourcePath).GetPath();
- TString path;
- if (filename) {
- path = Dir / filename;
- if (!NFs::SymLink(sourceAbsolutePath, path)) {
- ythrow TSystemError() << "Failed to create symlink for file " << sourceAbsolutePath.Quote() << " to file " << path.Quote();
- }
- } else {
- while (true) {
- path = Dir / RandGuid.GenGuid();
- if (NFs::SymLink(sourceAbsolutePath, path)) {
- break;
- } else if (LastSystemError() != EEXIST) {
- ythrow TSystemError() << "Failed to create symlink for file " << sourceAbsolutePath.Quote() << " to file " << path.Quote();
- }
- }
+ TString path;
+ if (filename) {
+ path = Dir / filename;
+ if (!NFs::SymLink(sourceAbsolutePath, path)) {
+ ythrow TSystemError() << "Failed to create symlink for file " << sourceAbsolutePath.Quote() << " to file " << path.Quote();
+ }
+ } else {
+ while (true) {
+ path = Dir / RandGuid.GenGuid();
+ if (NFs::SymLink(sourceAbsolutePath, path)) {
+ break;
+ } else if (LastSystemError() != EEXIST) {
+ ythrow TSystemError() << "Failed to create symlink for file " << sourceAbsolutePath.Quote() << " to file " << path.Quote();
+ }
+ }
Mapping.emplace(source, path);
}
return path;
@@ -67,19 +67,19 @@ THolder<TFilesBox> CreateFilesBox(const TFsPath& baseDir) {
TRandGuid randGuid;
TFsPath path = baseDir / randGuid.GenGuid();
- while (true) {
- if (!path.Exists()) {
- int r = Mkdir(path.c_str(), MODE0711);
- if (r == 0) {
- break;
- }
- if (LastSystemError() != EEXIST) {
- ythrow TIoSystemError() << "could not create directory " << path;
- }
- }
- path = baseDir / randGuid.GenGuid();
- }
-
+ while (true) {
+ if (!path.Exists()) {
+ int r = Mkdir(path.c_str(), MODE0711);
+ if (r == 0) {
+ break;
+ }
+ if (LastSystemError() != EEXIST) {
+ ythrow TIoSystemError() << "could not create directory " << path;
+ }
+ }
+ path = baseDir / randGuid.GenGuid();
+ }
+
return MakeHolder<TFilesBox>(std::move(path), std::move(randGuid));
}
diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp
index dc31e0ef30..6d591a5417 100644
--- a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp
+++ b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.cpp
@@ -1,35 +1,35 @@
-#include "yql_outproc_udf_resolver.h"
+#include "yql_outproc_udf_resolver.h"
#include "yql_simple_udf_resolver.h"
#include "yql_files_box.h"
-
+
#include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h>
#include <ydb/library/yql/providers/common/schema/expr/yql_expr_schema.h>
#include <ydb/library/yql/core/yql_holding_file_storage.h>
#include <ydb/library/yql/core/yql_type_annotation.h>
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/retry.h>
-
+
#include <ydb/library/yql/minikql/mkql_node.h>
#include <ydb/library/yql/minikql/mkql_type_builder.h>
#include <ydb/library/yql/minikql/mkql_program_builder.h>
#include <ydb/library/yql/minikql/mkql_utils.h>
-
-#include <library/cpp/protobuf/util/pb_io.h>
-
+
+#include <library/cpp/protobuf/util/pb_io.h>
+
#include <util/generic/scope.h>
-#include <util/stream/str.h>
-#include <util/string/strip.h>
-#include <util/system/shellcommand.h>
+#include <util/stream/str.h>
+#include <util/string/strip.h>
+#include <util/system/shellcommand.h>
#include <util/string/split.h>
-
+
#include <regex>
-namespace NYql {
-namespace NCommon {
-
-using namespace NKikimr;
-using namespace NKikimr::NMiniKQL;
-
+namespace NYql {
+namespace NCommon {
+
+using namespace NKikimr;
+using namespace NKikimr::NMiniKQL;
+
namespace {
template <typename F>
void RunResolver(
@@ -103,15 +103,15 @@ TString ExtractSharedObjectNameFromErrorMessage(const char* message) {
}
}
-class TOutProcUdfResolver : public IUdfResolver {
-public:
+class TOutProcUdfResolver : public IUdfResolver {
+public:
TOutProcUdfResolver(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
const TFileStoragePtr& fileStorage, const TString& resolverPath,
const TString& user, const TString& group, bool filterSyscalls,
const TString& udfDependencyStubPath, const TMap<TString, TString>& path2md5)
: FunctionRegistry_(functionRegistry)
, TypeInfoHelper_(new TTypeInfoHelper)
- , FileStorage_(fileStorage)
+ , FileStorage_(fileStorage)
, ResolverPath_(resolverPath)
, UdfDependencyStubPath_(udfDependencyStubPath)
, Path2Md5_(path2md5)
@@ -124,7 +124,7 @@ public:
UserGroupArgs_.push_back("-F");
}
}
-
+
TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override {
auto path = FunctionRegistry_->FindUdfPath(moduleName);
if (!path) {
@@ -133,22 +133,22 @@ public:
const TString md5 = Path2Md5_.Value(*path, "");
return MakeMaybe<TFilePathWithMd5>(*path, md5);
- }
-
+ }
+
bool LoadMetadata(const TVector<TImport*>& imports, const TVector<TFunction*>& functions, TExprContext& ctx) const override {
THashSet<TString> requiredLoadedModules;
THashSet<TString> requiredExternalModules;
TVector<TFunction*> loadedFunctions;
TVector<TFunction*> externalFunctions;
- bool hasErrors = false;
+ bool hasErrors = false;
for (auto udf : functions) {
- TStringBuf moduleName, funcName;
+ TStringBuf moduleName, funcName;
if (!SplitUdfName(udf->Name, moduleName, funcName) || moduleName.empty() || funcName.empty()) {
ctx.AddError(TIssue(udf->Pos, TStringBuilder() <<
- "Incorrect format of function name: " << udf->Name));
- hasErrors = true;
- } else {
+ "Incorrect format of function name: " << udf->Name));
+ hasErrors = true;
+ } else {
if (FunctionRegistry_->IsLoadedUdfModule(moduleName)) {
requiredLoadedModules.insert(TString(moduleName));
loadedFunctions.push_back(udf);
@@ -156,75 +156,75 @@ public:
requiredExternalModules.insert(TString(moduleName));
externalFunctions.push_back(udf);
}
- }
- }
-
- TResolve request;
+ }
+ }
+
+ TResolve request;
THashMap<TString, TImport*> importMap;
THoldingFileStorage holdingFileStorage(FileStorage_);
THolder<TFilesBox> filesBox = CreateFilesBoxOverFileStorageTemp();
-
+
THashMap<TString, TImport*> path2LoadedImport;
for (auto import : imports) {
- if (import->Modules) {
- bool needLibrary = false;
- for (auto& m : *import->Modules) {
+ if (import->Modules) {
+ bool needLibrary = false;
+ for (auto& m : *import->Modules) {
if (requiredLoadedModules.contains(m)) {
YQL_ENSURE(import->Block->Type == EUserDataType::PATH);
path2LoadedImport[import->Block->Data] = import;
}
if (requiredExternalModules.contains(m)) {
- needLibrary = true;
- break;
- }
- }
-
- if (!needLibrary) {
- continue;
- }
+ needLibrary = true;
+ break;
+ }
+ }
+
+ if (!needLibrary) {
+ continue;
+ }
} else {
import->Modules.ConstructInPlace();
- }
-
- importMap[import->FileAlias] = import;
-
- try {
+ }
+
+ importMap[import->FileAlias] = import;
+
+ try {
LoadImport(holdingFileStorage, *filesBox, *import, request);
} catch (const std::exception& e) {
ctx.AddError(ExceptionToIssue(e));
- hasErrors = true;
- }
- }
-
+ hasErrors = true;
+ }
+ }
+
for (auto& module : requiredExternalModules) {
if (auto path = FunctionRegistry_->FindUdfPath(module)) {
- auto importRequest = request.AddImports();
+ auto importRequest = request.AddImports();
const TString hiddenPath = filesBox->MakeLinkFrom(*path);
importRequest->SetFileAlias(hiddenPath);
importRequest->SetPath(hiddenPath);
- importRequest->SetSystem(true);
- }
- }
-
+ importRequest->SetSystem(true);
+ }
+ }
+
for (auto udf : externalFunctions) {
- auto udfRequest = request.AddUdfs();
- udfRequest->SetName(udf->Name);
- udfRequest->SetTypeConfig(udf->TypeConfig);
- if (udf->UserType) {
- udfRequest->SetUserType(WriteTypeToYson(udf->UserType));
- }
- }
-
+ auto udfRequest = request.AddUdfs();
+ udfRequest->SetName(udf->Name);
+ udfRequest->SetTypeConfig(udf->TypeConfig);
+ if (udf->UserType) {
+ udfRequest->SetUserType(WriteTypeToYson(udf->UserType));
+ }
+ }
+
TResolveResult response;
try {
response = RunResolverAndParseResult(request, { }, *filesBox);
filesBox->Destroy();
} catch (const std::exception& e) {
ctx.AddError(ExceptionToIssue(e));
- return false;
- }
+ return false;
+ }
// extract regardless of hasErrors value
hasErrors = !ExtractMetadata(response, importMap, externalFunctions, ctx) || hasErrors;
@@ -317,68 +317,68 @@ private:
static bool ExtractMetadata(const TResolveResult& response, const THashMap<TString, TImport*>& importMap, const TVector<TFunction*>& functions, TExprContext& ctx) {
bool hasErrors = false;
- YQL_ENSURE(response.UdfsSize() == functions.size(), "Number of returned udf signatures doesn't match original one");
-
- for (size_t i = 0; i < response.ImportsSize(); ++i) {
- const TImportResult& importRes = response.GetImports(i);
-
- TImport* import = nullptr;
- if (auto p = importMap.FindPtr(importRes.GetFileAlias())) {
- import = *p;
- }
- if (importRes.HasError()) {
+ YQL_ENSURE(response.UdfsSize() == functions.size(), "Number of returned udf signatures doesn't match original one");
+
+ for (size_t i = 0; i < response.ImportsSize(); ++i) {
+ const TImportResult& importRes = response.GetImports(i);
+
+ TImport* import = nullptr;
+ if (auto p = importMap.FindPtr(importRes.GetFileAlias())) {
+ import = *p;
+ }
+ if (importRes.HasError()) {
ctx.AddError(TIssue(import ? import->Pos : TPosition(), importRes.GetError()));
- hasErrors = true;
- } else if (import) {
- import->Modules.ConstructInPlace();
+ hasErrors = true;
+ } else if (import) {
+ import->Modules.ConstructInPlace();
for (auto& module : importRes.GetModules()) {
- import->Modules->push_back(module);
- }
- }
- }
-
- for (size_t i = 0; i < response.UdfsSize(); ++i) {
- TFunction* udf = functions[i];
- const TFunctionResult& udfRes = response.GetUdfs(i);
- if (udfRes.HasError()) {
+ import->Modules->push_back(module);
+ }
+ }
+ }
+
+ for (size_t i = 0; i < response.UdfsSize(); ++i) {
+ TFunction* udf = functions[i];
+ const TFunctionResult& udfRes = response.GetUdfs(i);
+ if (udfRes.HasError()) {
ctx.AddError(TIssue(udf->Pos, udfRes.GetError()));
- hasErrors = true;
- } else {
- udf->CallableType = ParseTypeFromYson(TStringBuf{udfRes.GetCallableType()}, ctx, udf->Pos);
- if (!udf->CallableType) {
- hasErrors = true;
- continue;
- }
- if (udfRes.HasNormalizedUserType()) {
- udf->NormalizedUserType = ParseTypeFromYson(TStringBuf{udfRes.GetNormalizedUserType()}, ctx, udf->Pos);
- if (!udf->NormalizedUserType) {
- hasErrors = true;
- continue;
- }
- }
- if (udfRes.HasRunConfigType()) {
- udf->RunConfigType = ParseTypeFromYson(TStringBuf{udfRes.GetRunConfigType()}, ctx, udf->Pos);
- if (!udf->RunConfigType) {
- hasErrors = true;
- continue;
- }
- }
- }
- }
-
- return !hasErrors;
- }
-
-private:
+ hasErrors = true;
+ } else {
+ udf->CallableType = ParseTypeFromYson(TStringBuf{udfRes.GetCallableType()}, ctx, udf->Pos);
+ if (!udf->CallableType) {
+ hasErrors = true;
+ continue;
+ }
+ if (udfRes.HasNormalizedUserType()) {
+ udf->NormalizedUserType = ParseTypeFromYson(TStringBuf{udfRes.GetNormalizedUserType()}, ctx, udf->Pos);
+ if (!udf->NormalizedUserType) {
+ hasErrors = true;
+ continue;
+ }
+ }
+ if (udfRes.HasRunConfigType()) {
+ udf->RunConfigType = ParseTypeFromYson(TStringBuf{udfRes.GetRunConfigType()}, ctx, udf->Pos);
+ if (!udf->RunConfigType) {
+ hasErrors = true;
+ continue;
+ }
+ }
+ }
+ }
+
+ return !hasErrors;
+ }
+
+private:
const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_;
NUdf::ITypeInfoHelper::TPtr TypeInfoHelper_;
- TFileStoragePtr FileStorage_;
+ TFileStoragePtr FileStorage_;
const TString ResolverPath_;
const TString UdfDependencyStubPath_;
TList<TString> UserGroupArgs_;
const TMap<TString, TString> Path2Md5_;
-};
-
+};
+
void LoadSystemModulePaths(
const TString& resolverPath,
const TString& dir,
@@ -400,7 +400,7 @@ void LoadSystemModulePaths(
});
}
-IUdfResolver::TPtr CreateOutProcUdfResolver(
+IUdfResolver::TPtr CreateOutProcUdfResolver(
const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
const TFileStoragePtr& fileStorage,
const TString& resolverPath,
@@ -410,7 +410,7 @@ IUdfResolver::TPtr CreateOutProcUdfResolver(
const TString& udfDependencyStubPath,
const TMap<TString, TString>& path2md5) {
return new TOutProcUdfResolver(functionRegistry, fileStorage, resolverPath, user, group, filterSyscalls, udfDependencyStubPath, path2md5);
-}
-
-} // namespace NCommon
-} // namespace NYql
+}
+
+} // namespace NCommon
+} // namespace NYql
diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.h b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.h
index 5659a833e4..10c42610d4 100644
--- a/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.h
+++ b/ydb/library/yql/providers/common/udf_resolve/yql_outproc_udf_resolver.h
@@ -1,22 +1,22 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/core/yql_udf_resolver.h>
#include <ydb/library/yql/core/file_storage/file_storage.h>
#include <ydb/library/yql/minikql/mkql_function_registry.h>
#include <ydb/library/yql/providers/common/proto/udf_resolver.pb.h>
-
+
#include <util/generic/map.h>
#include <util/generic/string.h>
-
-namespace NYql {
-namespace NCommon {
-
+
+namespace NYql {
+namespace NCommon {
+
void LoadSystemModulePaths(
const TString& resolverPath,
const TString& dir,
NKikimr::NMiniKQL::TUdfModulePathsMap* paths);
-IUdfResolver::TPtr CreateOutProcUdfResolver(
+IUdfResolver::TPtr CreateOutProcUdfResolver(
const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
const TFileStoragePtr& fileStorage,
const TString& resolverPath,
@@ -25,6 +25,6 @@ IUdfResolver::TPtr CreateOutProcUdfResolver(
bool filterSysCalls,
const TString& udfDependencyStubPath,
const TMap<TString, TString>& path2md5 = {});
-
-} // namespace NCommon
-} // namespace NYql
+
+} // namespace NCommon
+} // namespace NYql
diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp
index 3aac3487c0..43d9466a93 100644
--- a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp
+++ b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.cpp
@@ -1,5 +1,5 @@
#include "yql_simple_udf_resolver.h"
-
+
#include <ydb/library/yql/providers/common/mkql/yql_type_mkql.h>
#include <ydb/library/yql/core/yql_holding_file_storage.h>
@@ -8,10 +8,10 @@
#include <ydb/library/yql/minikql/mkql_program_builder.h>
#include <ydb/library/yql/minikql/mkql_utils.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
-
-#include <util/generic/vector.h>
-#include <util/generic/hash_set.h>
-#include <util/generic/hash.h>
+
+#include <util/generic/vector.h>
+#include <util/generic/hash_set.h>
+#include <util/generic/hash.h>
#include <util/generic/string.h>
#include <util/system/guard.h>
#include <util/system/spinlock.h>
@@ -24,17 +24,17 @@ using namespace NKikimr::NMiniKQL;
class TSimpleUdfResolver : public IUdfResolver {
public:
- TSimpleUdfResolver(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, const TFileStoragePtr& fileStorage, bool useFakeMD5)
+ TSimpleUdfResolver(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, const TFileStoragePtr& fileStorage, bool useFakeMD5)
: FunctionRegistry_(functionRegistry)
- , FileStorage_(fileStorage)
+ , FileStorage_(fileStorage)
, TypeInfoHelper_(new TTypeInfoHelper)
- , UseFakeMD5_(useFakeMD5)
+ , UseFakeMD5_(useFakeMD5)
{}
TMaybe<TFilePathWithMd5> GetSystemModulePath(const TStringBuf& moduleName) const override {
with_lock(Lock_) {
auto path = FunctionRegistry_->FindUdfPath(moduleName);
- return path ? MakeMaybe<TFilePathWithMd5>(*path, UseFakeMD5_ ? *path : TString()) : Nothing();
+ return path ? MakeMaybe<TFilePathWithMd5>(*path, UseFakeMD5_ ? *path : TString()) : Nothing();
}
}
@@ -59,10 +59,10 @@ public:
THoldingFileStorage holdingFileStorage(FileStorage_);
auto newRegistry = FunctionRegistry_->Clone();
THashMap<TString, TImport*> path2import;
- for (auto import: imports) {
- if (import->Modules) {
+ for (auto import: imports) {
+ if (import->Modules) {
bool needLibrary = false;
- for (auto& m : *import->Modules) {
+ for (auto& m : *import->Modules) {
if (requiredModules.contains(m)) {
needLibrary = true;
break;
@@ -72,30 +72,30 @@ public:
if (!needLibrary) {
continue;
}
- } else {
- import->Modules.ConstructInPlace();
+ } else {
+ import->Modules.ConstructInPlace();
}
try {
- if (FileStorage_) {
+ if (FileStorage_) {
auto link = holdingFileStorage.FreezeFile(*import->Block);
auto path = link->GetPath().GetPath();
newRegistry->LoadUdfs(path, {}, NUdf::IRegistrator::TFlags::TypesOnly);
path2import[path] = import;
- } else {
- if (import->Block->Type != EUserDataType::PATH) {
+ } else {
+ if (import->Block->Type != EUserDataType::PATH) {
ctx.AddError(TIssue(import->Pos, TStringBuilder() <<
- "Only path file type is supported, cannot load file with alias: " << import->FileAlias));
- hasErrors = true;
- continue;
- }
- newRegistry->LoadUdfs(import->Block->Data, {}, NUdf::IRegistrator::TFlags::TypesOnly);
- path2import[import->Block->Data] = import;
- }
+ "Only path file type is supported, cannot load file with alias: " << import->FileAlias));
+ hasErrors = true;
+ continue;
+ }
+ newRegistry->LoadUdfs(import->Block->Data, {}, NUdf::IRegistrator::TFlags::TypesOnly);
+ path2import[import->Block->Data] = import;
+ }
}
catch (yexception& e) {
ctx.AddError(TIssue(import->Pos, TStringBuilder()
- << "Internal error of loading udf module: " << import->FileAlias
+ << "Internal error of loading udf module: " << import->FileAlias
<< ", reason: " << e.what()));
hasErrors = true;
}
@@ -104,8 +104,8 @@ public:
if (!hasErrors) {
for (auto& m : newRegistry->GetAllModuleNames()) {
auto path = *newRegistry->FindUdfPath(m);
- if (auto import = path2import.FindPtr(path)) {
- (*import)->Modules->push_back(m);
+ if (auto import = path2import.FindPtr(path)) {
+ (*import)->Modules->push_back(m);
}
}
}
@@ -123,17 +123,17 @@ public:
private:
mutable TAdaptiveLock Lock_;
const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry_;
- TFileStoragePtr FileStorage_;
+ TFileStoragePtr FileStorage_;
NUdf::ITypeInfoHelper::TPtr TypeInfoHelper_;
- const bool UseFakeMD5_;
+ const bool UseFakeMD5_;
};
-IUdfResolver::TPtr CreateSimpleUdfResolver(
- const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- const TFileStoragePtr& fileStorage,
- bool useFakeMD5
-) {
- return new TSimpleUdfResolver(functionRegistry, fileStorage, useFakeMD5);
+IUdfResolver::TPtr CreateSimpleUdfResolver(
+ const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
+ const TFileStoragePtr& fileStorage,
+ bool useFakeMD5
+) {
+ return new TSimpleUdfResolver(functionRegistry, fileStorage, useFakeMD5);
}
bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions,
diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h
index 4aec038e43..910945cdd1 100644
--- a/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h
+++ b/ydb/library/yql/providers/common/udf_resolve/yql_simple_udf_resolver.h
@@ -10,8 +10,8 @@ namespace NYql {
namespace NCommon {
IUdfResolver::TPtr CreateSimpleUdfResolver(
- const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- const TFileStoragePtr& fileStorage = {}, bool useFakeMD5 = false);
+ const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
+ const TFileStoragePtr& fileStorage = {}, bool useFakeMD5 = false);
bool LoadFunctionsMetadata(const TVector<IUdfResolver::TFunction*>& functions,
const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry,
diff --git a/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp b/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp
index ac43a629b5..e7e2da10a0 100644
--- a/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp
+++ b/ydb/library/yql/providers/common/udf_resolve/yql_udf_resolver_with_index.cpp
@@ -161,14 +161,14 @@ private:
return false;
}
- function.CallableType = ParseTypeFromYson(TStringBuf{info.CallableType}, ctx, function.Pos);
+ function.CallableType = ParseTypeFromYson(TStringBuf{info.CallableType}, ctx, function.Pos);
if (!function.CallableType) {
ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Failed to build callable type from YSON for function " << function.Name));
return false;
}
if (info.RunConfigType) {
- function.RunConfigType = ParseTypeFromYson(TStringBuf{info.RunConfigType}, ctx, function.Pos);
+ function.RunConfigType = ParseTypeFromYson(TStringBuf{info.RunConfigType}, ctx, function.Pos);
if (!function.RunConfigType) {
ctx.AddError(TIssue(function.Pos, TStringBuilder() << "Failed to build run config type from YSON for function " << function.Name));
return false;
diff --git a/ydb/library/yql/providers/common/ya.make b/ydb/library/yql/providers/common/ya.make
index 66778b0bcf..f8a5fdd7e9 100644
--- a/ydb/library/yql/providers/common/ya.make
+++ b/ydb/library/yql/providers/common/ya.make
@@ -1,21 +1,21 @@
RECURSE(
codec
- config
- dq
+ config
+ dq
gateway
http_gateway
metrics
mkql
- comp_nodes
+ comp_nodes
proto
provider
- schema
- schema/expr
- schema/mkql
- schema/parser
- schema/skiff
+ schema
+ schema/expr
+ schema/mkql
+ schema/parser
+ schema/skiff
structured_token
token_accessor
- transform
- udf_resolve
+ transform
+ udf_resolve
)
diff --git a/ydb/library/yql/providers/config/yql_config_provider.cpp b/ydb/library/yql/providers/config/yql_config_provider.cpp
index 6952e96cf0..b3cc89946c 100644
--- a/ydb/library/yql/providers/config/yql_config_provider.cpp
+++ b/ydb/library/yql/providers/config/yql_config_provider.cpp
@@ -1,5 +1,5 @@
#include "yql_config_provider.h"
-
+
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/providers/common/provider/yql_data_provider_impl.h>
#include <ydb/library/yql/providers/common/proto/gateways_config.pb.h>
@@ -13,13 +13,13 @@
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/fetch/fetch.h>
#include <ydb/library/yql/utils/retry.h>
-
+
#include <library/cpp/json/json_reader.h>
#include <util/string/cast.h>
-#include <vector>
-
+#include <vector>
+
namespace NYql {
namespace {
@@ -47,7 +47,7 @@ namespace {
NYson::EYsonFormat ysonFormat = NCommon::GetYsonFormat(fillSettings);
auto nodeToPull = input->Child(0)->Child(0);
- if (nodeToPull->IsCallable(ConfReadName)) {
+ if (nodeToPull->IsCallable(ConfReadName)) {
auto key = nodeToPull->Child(2);
auto tag = key->Child(0)->Child(0)->Content();
if (tag == "data_sinks" || tag == "data_sources") {
@@ -100,11 +100,11 @@ namespace {
}
if (input->Content() == ConfigureName) {
- auto requireStatus = RequireChild(*input, 0);
- if (requireStatus.Level != TStatus::Ok) {
- return requireStatus;
- }
-
+ auto requireStatus = RequireChild(*input, 0);
+ if (requireStatus.Level != TStatus::Ok) {
+ return requireStatus;
+ }
+
input->SetState(TExprNode::EState::ExecutionComplete);
input->SetResult(ctx.NewWorld(input->Pos()));
return TStatus::Ok;
@@ -118,19 +118,19 @@ namespace {
const TTypeAnnotationContext& Types;
};
- class TConfigProvider : public TDataProviderBase {
+ class TConfigProvider : public TDataProviderBase {
public:
struct TFunctions {
THashSet<TStringBuf> Names;
TFunctions() {
- Names.insert(ConfReadName);
+ Names.insert(ConfReadName);
}
};
TConfigProvider(TTypeAnnotationContext& types, const TGatewaysConfig* config, const TAllowSettingPolicy& policy)
: Types(types)
- , CoreConfig(config && config->HasYqlCore() ? &config->GetYqlCore() : nullptr)
+ , CoreConfig(config && config->HasYqlCore() ? &config->GetYqlCore() : nullptr)
, Policy(policy)
{}
@@ -138,23 +138,23 @@ namespace {
return ConfigProviderName;
}
- bool Initialize(TExprContext& ctx) override {
- if (CoreConfig) {
- TPosition pos;
- for (auto& flag: CoreConfig->GetFlags()) {
- TVector<TStringBuf> args;
- for (auto& arg: flag.GetArgs()) {
- args.push_back(arg);
- }
- if (!ApplyFlag(pos, flag.GetName(), args, ctx)) {
- return false;
- }
- }
- }
- return true;
- }
-
- bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
+ bool Initialize(TExprContext& ctx) override {
+ if (CoreConfig) {
+ TPosition pos;
+ for (auto& flag: CoreConfig->GetFlags()) {
+ TVector<TStringBuf> args;
+ for (auto& arg: flag.GetArgs()) {
+ args.push_back(arg);
+ }
+ if (!ApplyFlag(pos, flag.GetName(), args, ctx)) {
+ return false;
+ }
+ }
+ }
+ return true;
+ }
+
+ bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
if (!EnsureArgsCount(node, 1, ctx)) {
return false;
}
@@ -169,7 +169,7 @@ namespace {
bool CanParse(const TExprNode& node) override {
if (ConfigProviderFunctions().contains(node.Content()) ||
- node.Content() == ConfigureName)
+ node.Content() == ConfigureName)
{
return MatchCategory(node);
}
@@ -193,10 +193,10 @@ namespace {
TOptimizeExprSettings settings(nullptr);
settings.VisitChanges = true;
auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) -> TExprNode::TPtr {
- auto res = node;
+ auto res = node;
if (!hasPendingEvaluations && node->Content() == ConfigureName) {
if (!EnsureMinArgsCount(*node, 2, ctx)) {
- return {};
+ return {};
}
if (!node->Child(1)->IsCallable("DataSource")) {
@@ -208,38 +208,38 @@ namespace {
}
if (!EnsureMinArgsCount(*node, 3, ctx)) {
- return {};
+ return {};
}
if (!EnsureAtom(*node->Child(2), ctx)) {
- return {};
+ return {};
}
- TStringBuf command = node->Child(2)->Content();
- TVector<TStringBuf> args;
- for (size_t i = 3; i < node->ChildrenSize(); ++i) {
- if (node->Child(i)->IsCallable("EvaluateAtom")) {
+ TStringBuf command = node->Child(2)->Content();
+ TVector<TStringBuf> args;
+ for (size_t i = 3; i < node->ChildrenSize(); ++i) {
+ if (node->Child(i)->IsCallable("EvaluateAtom")) {
hasPendingEvaluations = true;
- return res;
+ return res;
}
- if (!EnsureAtom(*node->Child(i), ctx)) {
+ if (!EnsureAtom(*node->Child(i), ctx)) {
return {};
}
- args.push_back(node->Child(i)->Content());
+ args.push_back(node->Child(i)->Content());
}
- if (!ApplyFlag(ctx.GetPosition(node->Child(2)->Pos()), command, args, ctx)) {
- return {};
+ if (!ApplyFlag(ctx.GetPosition(node->Child(2)->Pos()), command, args, ctx)) {
+ return {};
}
- if (command == "PureDataSource") {
- if (Types.PureResultDataSource != node->Child(3)->Content()) {
- res = ctx.ChangeChild(*node, 3, ctx.RenameNode(*node->Child(3), Types.PureResultDataSource));
+ if (command == "PureDataSource") {
+ if (Types.PureResultDataSource != node->Child(3)->Content()) {
+ res = ctx.ChangeChild(*node, 3, ctx.RenameNode(*node->Child(3), Types.PureResultDataSource));
}
- }
+ }
}
- return res;
+ return res;
}, ctx, settings);
return status;
@@ -295,7 +295,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- if (!input->Child(3)->GetTypeAnn() || !input->Child(3)->IsComposable()) {
+ if (!input->Child(3)->GetTypeAnn() || !input->Child(3)->IsComposable()) {
ctx.AddError(TIssue(ctx.GetPosition(input->Child(3)->Pos()), "Expected composable data"));
return IGraphTransformer::TStatus::Error;
}
@@ -313,7 +313,7 @@ namespace {
auto stringAnnotation = ctx.MakeType<TDataExprType>(EDataSlot::String);
auto listOfString = ctx.MakeType<TListExprType>(stringAnnotation);
TTypeAnnotationNode::TListType children;
- children.push_back(input->Child(0)->GetTypeAnn());
+ children.push_back(input->Child(0)->GetTypeAnn());
if (tag == "data_sources" || tag == "data_sinks") {
children.push_back(listOfString);
} else {
@@ -322,7 +322,7 @@ namespace {
}
auto tupleAnn = ctx.MakeType<TTupleExprType>(children);
- input->SetTypeAnn(tupleAnn);
+ input->SetTypeAnn(tupleAnn);
return IGraphTransformer::TStatus::Ok;
}
else if (input->Content() == ConfigureName) {
@@ -330,7 +330,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(input->Child(0)->GetTypeAnn());
+ input->SetTypeAnn(input->Child(0)->GetTypeAnn());
return IGraphTransformer::TStatus::Ok;
}
@@ -345,8 +345,8 @@ namespace {
TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override {
auto read = node->Child(0);
TString newName;
- if (read->Content() == ReadName) {
- newName = ConfReadName;
+ if (read->Content() == ReadName) {
+ newName = ConfReadName;
}
else {
YQL_ENSURE(false, "Expected Read!");
@@ -374,7 +374,7 @@ namespace {
bool CanExecute(const TExprNode& node) override {
if (ConfigProviderFunctions().contains(node.Content()) ||
- node.Content() == ConfigureName)
+ node.Content() == ConfigureName)
{
return MatchCategory(node);
}
@@ -420,7 +420,7 @@ namespace {
return true;
}
- bool ApplyFlag(const TPosition& pos, const TStringBuf name, const TVector<TStringBuf>& args, TExprContext& ctx) {
+ bool ApplyFlag(const TPosition& pos, const TStringBuf name, const TVector<TStringBuf>& args, TExprContext& ctx) {
if (!IsSettingAllowed(pos, name, ctx)) {
return false;
}
@@ -430,233 +430,233 @@ namespace {
return false;
}
} else if (name == "ImportUdfs") {
- if (!ImportUdfs(pos, args, ctx)) {
- return false;
- }
- } else if (name == "AddFileByUrl") {
- if (!AddFileByUrl(pos, args, ctx)) {
- return false;
- }
- } else if (name == "AddFolderByUrl") {
- if (!AddFolderByUrl(pos, args, ctx)) {
- return false;
- }
- } else if (name == "SetPackageVersion") {
- if (!SetPackageVersion(pos, args, ctx)) {
- return false;
- }
- }
- else if (name == "ValidateUdf") {
- if (args.size() != 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
- return false;
- }
-
- try {
- Types.ValidateMode = NKikimr::NUdf::ValidateModeByStr(TString(args[0]));
- } catch (const yexception& err) {
- ctx.AddError(TIssue(pos, TStringBuilder() << err.AsStrBuf() << ", available modes: " << NKikimr::NUdf::ValidateModeAvailables()));
- return false;
- }
- }
- else if (name == "LLVM_OFF") {
- if (args.size() != 0) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
- return false;
- }
- Types.OptLLVM = "OFF";
- }
- else if (name == "LLVM") {
- if (args.size() > 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size()));
- return false;
- }
- Types.OptLLVM = args.empty() ? TString() : TString(args[0]);
- }
- else if (name == "NodesAllocationLimit") {
- if (args.size() != 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
- return false;
- }
- if (!TryFromString(args[0], ctx.NodesAllocationLimit)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected integer, but got: " << args[0]));
- return false;
- }
- }
- else if (name == "StringsAllocationLimit") {
- if (args.size() != 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
- return false;
- }
- if (!TryFromString(args[0], ctx.StringsAllocationLimit)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected integer, but got: " << args[0]));
- return false;
- }
- }
- else if (name == "RepeatTransformLimit") {
- if (args.size() != 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
- return false;
- }
- if (!TryFromString(args[0], ctx.RepeatTransformLimit)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected integer, but got: " << args[0]));
- return false;
- }
- }
- else if (name == "PureDataSource") {
- if (args.size() != 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
- return false;
- }
-
- auto dataSource = args[0];
- if (Find(Types.AvailablePureResultDataSources, dataSource) == Types.AvailablePureResultDataSources.end()) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Unsupported datasource for result provider: " << dataSource));
- return false;
- }
- if (auto p = Types.DataSourceMap.FindPtr(dataSource)) {
- if ((*p)->GetName() != dataSource) {
- dataSource = (*p)->GetName();
- }
- } else {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Unknown datasource for result provider: " << dataSource));
- return false;
- }
-
- Types.PureResultDataSource = dataSource;
- }
- else if (name == "FullResultDataSink") {
- if (args.size() != 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
- return false;
- }
-
- auto dataSink = args[0];
- if (auto p = Types.DataSinkMap.FindPtr(dataSink)) {
- if ((*p)->GetName() != dataSink) {
- dataSink = (*p)->GetName();
- }
- } else {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Unknown datasink for full result provider: " << dataSink));
- return false;
- }
-
- Types.FullResultDataSink = dataSink;
- }
- else if (name == "Diagnostics") {
- if (args.size() != 0) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
- return false;
- }
-
- Types.Diagnostics = true;
- }
+ if (!ImportUdfs(pos, args, ctx)) {
+ return false;
+ }
+ } else if (name == "AddFileByUrl") {
+ if (!AddFileByUrl(pos, args, ctx)) {
+ return false;
+ }
+ } else if (name == "AddFolderByUrl") {
+ if (!AddFolderByUrl(pos, args, ctx)) {
+ return false;
+ }
+ } else if (name == "SetPackageVersion") {
+ if (!SetPackageVersion(pos, args, ctx)) {
+ return false;
+ }
+ }
+ else if (name == "ValidateUdf") {
+ if (args.size() != 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
+ return false;
+ }
+
+ try {
+ Types.ValidateMode = NKikimr::NUdf::ValidateModeByStr(TString(args[0]));
+ } catch (const yexception& err) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << err.AsStrBuf() << ", available modes: " << NKikimr::NUdf::ValidateModeAvailables()));
+ return false;
+ }
+ }
+ else if (name == "LLVM_OFF") {
+ if (args.size() != 0) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
+ return false;
+ }
+ Types.OptLLVM = "OFF";
+ }
+ else if (name == "LLVM") {
+ if (args.size() > 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size()));
+ return false;
+ }
+ Types.OptLLVM = args.empty() ? TString() : TString(args[0]);
+ }
+ else if (name == "NodesAllocationLimit") {
+ if (args.size() != 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
+ return false;
+ }
+ if (!TryFromString(args[0], ctx.NodesAllocationLimit)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected integer, but got: " << args[0]));
+ return false;
+ }
+ }
+ else if (name == "StringsAllocationLimit") {
+ if (args.size() != 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
+ return false;
+ }
+ if (!TryFromString(args[0], ctx.StringsAllocationLimit)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected integer, but got: " << args[0]));
+ return false;
+ }
+ }
+ else if (name == "RepeatTransformLimit") {
+ if (args.size() != 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
+ return false;
+ }
+ if (!TryFromString(args[0], ctx.RepeatTransformLimit)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected integer, but got: " << args[0]));
+ return false;
+ }
+ }
+ else if (name == "PureDataSource") {
+ if (args.size() != 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
+ return false;
+ }
+
+ auto dataSource = args[0];
+ if (Find(Types.AvailablePureResultDataSources, dataSource) == Types.AvailablePureResultDataSources.end()) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Unsupported datasource for result provider: " << dataSource));
+ return false;
+ }
+ if (auto p = Types.DataSourceMap.FindPtr(dataSource)) {
+ if ((*p)->GetName() != dataSource) {
+ dataSource = (*p)->GetName();
+ }
+ } else {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Unknown datasource for result provider: " << dataSource));
+ return false;
+ }
+
+ Types.PureResultDataSource = dataSource;
+ }
+ else if (name == "FullResultDataSink") {
+ if (args.size() != 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
+ return false;
+ }
+
+ auto dataSink = args[0];
+ if (auto p = Types.DataSinkMap.FindPtr(dataSink)) {
+ if ((*p)->GetName() != dataSink) {
+ dataSink = (*p)->GetName();
+ }
+ } else {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Unknown datasink for full result provider: " << dataSink));
+ return false;
+ }
+
+ Types.FullResultDataSink = dataSink;
+ }
+ else if (name == "Diagnostics") {
+ if (args.size() != 0) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
+ return false;
+ }
+
+ Types.Diagnostics = true;
+ }
else if (name == TStringBuf("Warning")) {
- if (!SetWarningRule(pos, args, ctx)) {
- return false;
- }
- }
- else if (name == "UdfSupportsYield") {
- if (args.size() > 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size()));
- return false;
- }
-
- bool res = true;
- if (!args.empty()) {
- if (!TryFromString(args[0], res)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected bool, but got: " << args[0]));
- return false;
- }
- }
-
- Types.UdfSupportsYield = res;
- }
- else if (name == "EvaluateForLimit") {
- if (args.size() != 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
- return false;
- }
- if (!TryFromString(args[0], Types.EvaluateForLimit)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected integer, but got: " << args[0]));
- return false;
- }
- }
- else if (name == "DisablePullUpFlatMapOverJoin" || name == "PullUpFlatMapOverJoin") {
- if (args.size() != 0) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
- return false;
- }
-
- Types.PullUpFlatMapOverJoin = (name == "PullUpFlatMapOverJoin");
- }
- else if (name == "SQL") {
- if (args.size() > 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size()));
- return false;
- }
-
- Types.DeprecatedSQL = (args[0] == "0");
- }
- else if (name == "DisableConstraintCheck") {
- if (args.empty()) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at least 1 argument, but got " << args.size()));
- return false;
- }
- for (auto arg: args) {
- Types.DisableConstraintCheck.emplace(arg);
- }
- }
- else if (name == "EnableConstraintCheck") {
- if (args.empty()) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at least 1 argument, but got " << args.size()));
- return false;
- }
- for (auto arg: args) {
+ if (!SetWarningRule(pos, args, ctx)) {
+ return false;
+ }
+ }
+ else if (name == "UdfSupportsYield") {
+ if (args.size() > 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size()));
+ return false;
+ }
+
+ bool res = true;
+ if (!args.empty()) {
+ if (!TryFromString(args[0], res)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected bool, but got: " << args[0]));
+ return false;
+ }
+ }
+
+ Types.UdfSupportsYield = res;
+ }
+ else if (name == "EvaluateForLimit") {
+ if (args.size() != 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
+ return false;
+ }
+ if (!TryFromString(args[0], Types.EvaluateForLimit)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected integer, but got: " << args[0]));
+ return false;
+ }
+ }
+ else if (name == "DisablePullUpFlatMapOverJoin" || name == "PullUpFlatMapOverJoin") {
+ if (args.size() != 0) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
+ return false;
+ }
+
+ Types.PullUpFlatMapOverJoin = (name == "PullUpFlatMapOverJoin");
+ }
+ else if (name == "SQL") {
+ if (args.size() > 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size()));
+ return false;
+ }
+
+ Types.DeprecatedSQL = (args[0] == "0");
+ }
+ else if (name == "DisableConstraintCheck") {
+ if (args.empty()) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at least 1 argument, but got " << args.size()));
+ return false;
+ }
+ for (auto arg: args) {
+ Types.DisableConstraintCheck.emplace(arg);
+ }
+ }
+ else if (name == "EnableConstraintCheck") {
+ if (args.empty()) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at least 1 argument, but got " << args.size()));
+ return false;
+ }
+ for (auto arg: args) {
Types.DisableConstraintCheck.erase(TString{arg});
- }
- }
- else if (name == "DisableConstraints") {
- if (args.empty()) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at least 1 argument, but got " << args.size()));
- return false;
- }
- for (auto arg: args) {
- ctx.DisabledConstraints.emplace(arg);
- }
- }
- else if (name == "EnableConstraints") {
- if (args.empty()) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at least 1 argument, but got " << args.size()));
- return false;
- }
- for (auto arg: args) {
- ctx.DisabledConstraints.erase(arg);
- }
- }
- else if (name == "UseTableMetaFromGraph") {
- if (args.size() > 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size()));
- return false;
- }
-
- bool res = true;
- if (!args.empty()) {
- if (!TryFromString(args[0], res)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected bool, but got: " << args[0]));
- return false;
- }
- }
-
- Types.UseTableMetaFromGraph = res;
- }
- else if (name == "DiscoveryMode") {
- if (args.size() != 0) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
- return false;
- }
- Types.DiscoveryMode = true;
- }
+ }
+ }
+ else if (name == "DisableConstraints") {
+ if (args.empty()) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at least 1 argument, but got " << args.size()));
+ return false;
+ }
+ for (auto arg: args) {
+ ctx.DisabledConstraints.emplace(arg);
+ }
+ }
+ else if (name == "EnableConstraints") {
+ if (args.empty()) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at least 1 argument, but got " << args.size()));
+ return false;
+ }
+ for (auto arg: args) {
+ ctx.DisabledConstraints.erase(arg);
+ }
+ }
+ else if (name == "UseTableMetaFromGraph") {
+ if (args.size() > 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected at most 1 argument, but got " << args.size()));
+ return false;
+ }
+
+ bool res = true;
+ if (!args.empty()) {
+ if (!TryFromString(args[0], res)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected bool, but got: " << args[0]));
+ return false;
+ }
+ }
+
+ Types.UseTableMetaFromGraph = res;
+ }
+ else if (name == "DiscoveryMode") {
+ if (args.size() != 0) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
+ return false;
+ }
+ Types.DiscoveryMode = true;
+ }
else if (name == "EnableSystemColumns") {
if (args.size() != 0) {
ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
@@ -683,32 +683,32 @@ namespace {
return false;
}
}
- else if (name == "IssueCountLimit") {
- if (args.size() != 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
- return false;
- }
- size_t limit = 0;
- if (!TryFromString(args[0], limit)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected unsigned integer, but got: " << args[0]));
- return false;
- }
- ctx.IssueManager.SetIssueCountLimit(limit);
- }
- else if (name == "StrictTableProps") {
- if (args.size() != 0) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
- return false;
- }
- Types.StrictTableProps = true;
- }
- else if (name == "DisableStrictTableProps") {
- if (args.size() != 0) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
- return false;
- }
- Types.StrictTableProps = false;
- }
+ else if (name == "IssueCountLimit") {
+ if (args.size() != 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
+ return false;
+ }
+ size_t limit = 0;
+ if (!TryFromString(args[0], limit)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected unsigned integer, but got: " << args[0]));
+ return false;
+ }
+ ctx.IssueManager.SetIssueCountLimit(limit);
+ }
+ else if (name == "StrictTableProps") {
+ if (args.size() != 0) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
+ return false;
+ }
+ Types.StrictTableProps = true;
+ }
+ else if (name == "DisableStrictTableProps") {
+ if (args.size() != 0) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected no arguments, but got " << args.size()));
+ return false;
+ }
+ Types.StrictTableProps = false;
+ }
else if (name == "GeobaseDownloadUrl") {
if (args.size() != 1) {
ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
@@ -735,52 +735,52 @@ namespace {
Types.OrderedColumns = (name == "OrderedColumns");
}
- else if (name == "FolderSubDirsLimit") {
- if (args.size() != 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
- return false;
- }
- if (!TryFromString(args[0], Types.FolderSubDirsLimit)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected integer, but got: " << args[0]));
- return false;
- }
- }
+ else if (name == "FolderSubDirsLimit") {
+ if (args.size() != 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
+ return false;
+ }
+ if (!TryFromString(args[0], Types.FolderSubDirsLimit)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected integer, but got: " << args[0]));
+ return false;
+ }
+ }
else {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Unsupported command: " << name));
- return false;
- }
- return true;
- }
-
- bool ImportUdfs(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
- if (args.size() != 1) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
- return false;
- }
-
- if (Types.DisableNativeUdfSupport) {
- ctx.AddError(TIssue(pos, "Native UDF support is disabled"));
- return false;
- }
-
- // file alias
- const auto& fileAlias = args[0];
- const auto key = TUserDataStorage::ComposeUserDataKey(fileAlias);
- TString errorMessage;
- const TUserDataBlock* udfSource = Types.UserDataStorage->FreezeUdfNoThrow(key, errorMessage);
- if (!udfSource) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Unknown file: " << fileAlias << ", details: " << errorMessage));
- return false;
- }
-
- IUdfResolver::TImport import;
- import.Pos = pos;
- import.FileAlias = fileAlias;
- import.Block = udfSource;
- Types.UdfImports.insert({ TString(fileAlias), import });
- return true;
- }
-
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Unsupported command: " << name));
+ return false;
+ }
+ return true;
+ }
+
+ bool ImportUdfs(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
+ if (args.size() != 1) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 1 argument, but got " << args.size()));
+ return false;
+ }
+
+ if (Types.DisableNativeUdfSupport) {
+ ctx.AddError(TIssue(pos, "Native UDF support is disabled"));
+ return false;
+ }
+
+ // file alias
+ const auto& fileAlias = args[0];
+ const auto key = TUserDataStorage::ComposeUserDataKey(fileAlias);
+ TString errorMessage;
+ const TUserDataBlock* udfSource = Types.UserDataStorage->FreezeUdfNoThrow(key, errorMessage);
+ if (!udfSource) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Unknown file: " << fileAlias << ", details: " << errorMessage));
+ return false;
+ }
+
+ IUdfResolver::TImport import;
+ import.Pos = pos;
+ import.FileAlias = fileAlias;
+ import.Block = udfSource;
+ Types.UdfImports.insert({ TString(fileAlias), import });
+ return true;
+ }
+
bool AddCredential(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
if (args.size() != 4) {
ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 4 arguments, but got " << args.size()));
@@ -800,200 +800,200 @@ namespace {
return true;
}
- bool AddFileByUrlImpl(const TStringBuf alias, const TStringBuf url, const TPosition pos, TExprContext& ctx) {
- if (url.empty()) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Empty URL for file '" << alias << "'."));
- return false;
- }
-
- auto key = TUserDataStorage::ComposeUserDataKey(alias);
- if (Types.UserDataStorage->ContainsUserDataBlock(key)) {
- // Don't overwrite.
- return true;
- }
-
- TUserDataBlock block;
- block.Type = EUserDataType::URL;
- block.Data = url;
- Types.UserDataStorage->TryFillUserDataToken(block);
- Types.UserDataStorage->AddUserDataBlock(key, block);
- return true;
- }
-
- bool AddFileByUrl(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
- if (args.size() != 2) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 2 arguments, but got " << args.size()));
- return false;
- }
-
- return AddFileByUrlImpl(args[0], args[1], pos, ctx);
- }
-
- bool SetPackageVersion(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
- if (args.size() != 2) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 2 arguments, but got " << args.size()));
- return false;
- }
-
- ui32 version = 0;
- if (!TryFromString(args[1], version)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Unable to parse package version from " << args[1]));
- return false;
- }
-
- if (!Types.UdfIndexPackageSet || !Types.UdfIndex) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "UdfIndex is not initialized, unable to set version for package " << args[0]));
- return false;
- }
-
- if (!Types.UdfIndexPackageSet->AddResourceTo(TString(args[0]), version, Types.UdfIndex)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Unable set default version to " << version << " for package " << args[0]));
- return false;
- }
-
- return true;
- }
-
- bool DoListSandboxFolder(const TStringBuf url, const TPosition& pos, TExprContext& ctx, NJson::TJsonValue& content) {
+ bool AddFileByUrlImpl(const TStringBuf alias, const TStringBuf url, const TPosition pos, TExprContext& ctx) {
+ if (url.empty()) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Empty URL for file '" << alias << "'."));
+ return false;
+ }
+
+ auto key = TUserDataStorage::ComposeUserDataKey(alias);
+ if (Types.UserDataStorage->ContainsUserDataBlock(key)) {
+ // Don't overwrite.
+ return true;
+ }
+
+ TUserDataBlock block;
+ block.Type = EUserDataType::URL;
+ block.Data = url;
+ Types.UserDataStorage->TryFillUserDataToken(block);
+ Types.UserDataStorage->AddUserDataBlock(key, block);
+ return true;
+ }
+
+ bool AddFileByUrl(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
+ if (args.size() != 2) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 2 arguments, but got " << args.size()));
+ return false;
+ }
+
+ return AddFileByUrlImpl(args[0], args[1], pos, ctx);
+ }
+
+ bool SetPackageVersion(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
+ if (args.size() != 2) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 2 arguments, but got " << args.size()));
+ return false;
+ }
+
+ ui32 version = 0;
+ if (!TryFromString(args[1], version)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Unable to parse package version from " << args[1]));
+ return false;
+ }
+
+ if (!Types.UdfIndexPackageSet || !Types.UdfIndex) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "UdfIndex is not initialized, unable to set version for package " << args[0]));
+ return false;
+ }
+
+ if (!Types.UdfIndexPackageSet->AddResourceTo(TString(args[0]), version, Types.UdfIndex)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Unable set default version to " << version << " for package " << args[0]));
+ return false;
+ }
+
+ return true;
+ }
+
+ bool DoListSandboxFolder(const TStringBuf url, const TPosition& pos, TExprContext& ctx, NJson::TJsonValue& content) {
TString urlStr(url);
if (!url.empty() && url.back() != '/') {
urlStr += "/";
}
const THttpURL& httpUrl = ParseURL(urlStr);
- if (httpUrl.GetHost() != "proxy.sandbox.yandex-team.ru") {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Adding folder by URL is currently supported only for proxy.sandbox.yandex-team.ru. Host " << httpUrl.GetHost() << " is not supported"));
- return false;
- }
-
- THttpHeaders headers;
- headers.AddHeader("Accept", "application/json");
- auto result = Fetch(httpUrl, headers, TDuration::Seconds(30));
- if (!result) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Failed to fetch " << url << " for building folder"));
- return false;
- }
- if (result->GetRetCode() != 200) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Failed to fetch " << url << " for building folder (http code: " << result->GetRetCode() << ")"));
- return false;
- }
-
- if (!NJson::ReadJsonTree(result->GetStream().ReadAll(), &content)) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Failed to parse json from " << url << " for building folder"));
- return false;
- }
-
- return true;
- }
-
- bool ListSandboxFolder(const TStringBuf url, const TPosition& pos, TExprContext& ctx, NJson::TJsonValue& content) {
- try {
-
- return WithRetry<std::exception>(3, [&]() {
- return DoListSandboxFolder(url, pos, ctx, content);
- }, [&](const auto& e, int attempt, int attemptCount) {
- YQL_CLOG(WARN, ProviderConfig) << "Error in loading sandbox folder " << url << ", attempt " << attempt << "/" << attemptCount << ", details: " << e.what();
- });
-
- } catch (const std::exception& e) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Failed to load sandbox folder content from " << url << ", details: " << e.what()));
- return false;
- }
- }
-
- static TString MakeHttps(const TString& url) {
- if (url.StartsWith("http:")) {
- return "https:" + url.substr(5);
- }
- return url;
- }
-
- bool AddFolderByUrl(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
- if (args.size() != 2) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 2 arguments, but got " << args.size()));
- return false;
- }
-
- std::vector<std::pair<TString, TString>> queue;
- queue.emplace_back(args[0], args[1]);
-
- size_t count = 0;
- while (!queue.empty()) {
- auto [prefix, url] = queue.back();
- queue.pop_back();
-
- YQL_CLOG(DEBUG, ProviderConfig) << "Listing sandbox folder " << prefix << ": " << url;
- NJson::TJsonValue content;
- if (!ListSandboxFolder(url, pos, ctx, content)) {
- return false;
- }
-
- for (const auto& file : content.GetMap()) {
- const auto& fileAttrs = file.second.GetMap();
- auto fileUrl = fileAttrs.FindPtr("url");
- if (fileUrl) {
- TString type = "REGULAR";
- if (auto t = fileAttrs.FindPtr("type")) {
- type = t->GetString();
- }
- TStringBuilder alias;
- if (!prefix.empty()) {
- alias << prefix << "/";
- }
- alias << file.first;
- if (type == "REGULAR") {
- if (!AddFileByUrlImpl(alias, TStringBuf(MakeHttps(fileUrl->GetString())), pos, ctx)) {
- return false;
- }
- } else if (type == "DIRECTORY") {
- queue.emplace_back(alias, fileUrl->GetString());
- if (++count > Types.FolderSubDirsLimit) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Sandbox resource has too many subfolders. Limit is " << Types.FolderSubDirsLimit));
- return false;
- }
- } else {
- YQL_CLOG(WARN, ProviderConfig) << "Got unknown sandbox item type: " << type << ", name=" << alias;
- }
- }
- }
- }
-
- return true;
-
- }
-
- bool SetWarningRule(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
- if (args.size() != 2) {
- ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 2 arguments, but got " << args.size()));
- return false;
- }
-
+ if (httpUrl.GetHost() != "proxy.sandbox.yandex-team.ru") {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Adding folder by URL is currently supported only for proxy.sandbox.yandex-team.ru. Host " << httpUrl.GetHost() << " is not supported"));
+ return false;
+ }
+
+ THttpHeaders headers;
+ headers.AddHeader("Accept", "application/json");
+ auto result = Fetch(httpUrl, headers, TDuration::Seconds(30));
+ if (!result) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Failed to fetch " << url << " for building folder"));
+ return false;
+ }
+ if (result->GetRetCode() != 200) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Failed to fetch " << url << " for building folder (http code: " << result->GetRetCode() << ")"));
+ return false;
+ }
+
+ if (!NJson::ReadJsonTree(result->GetStream().ReadAll(), &content)) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Failed to parse json from " << url << " for building folder"));
+ return false;
+ }
+
+ return true;
+ }
+
+ bool ListSandboxFolder(const TStringBuf url, const TPosition& pos, TExprContext& ctx, NJson::TJsonValue& content) {
+ try {
+
+ return WithRetry<std::exception>(3, [&]() {
+ return DoListSandboxFolder(url, pos, ctx, content);
+ }, [&](const auto& e, int attempt, int attemptCount) {
+ YQL_CLOG(WARN, ProviderConfig) << "Error in loading sandbox folder " << url << ", attempt " << attempt << "/" << attemptCount << ", details: " << e.what();
+ });
+
+ } catch (const std::exception& e) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Failed to load sandbox folder content from " << url << ", details: " << e.what()));
+ return false;
+ }
+ }
+
+ static TString MakeHttps(const TString& url) {
+ if (url.StartsWith("http:")) {
+ return "https:" + url.substr(5);
+ }
+ return url;
+ }
+
+ bool AddFolderByUrl(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
+ if (args.size() != 2) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 2 arguments, but got " << args.size()));
+ return false;
+ }
+
+ std::vector<std::pair<TString, TString>> queue;
+ queue.emplace_back(args[0], args[1]);
+
+ size_t count = 0;
+ while (!queue.empty()) {
+ auto [prefix, url] = queue.back();
+ queue.pop_back();
+
+ YQL_CLOG(DEBUG, ProviderConfig) << "Listing sandbox folder " << prefix << ": " << url;
+ NJson::TJsonValue content;
+ if (!ListSandboxFolder(url, pos, ctx, content)) {
+ return false;
+ }
+
+ for (const auto& file : content.GetMap()) {
+ const auto& fileAttrs = file.second.GetMap();
+ auto fileUrl = fileAttrs.FindPtr("url");
+ if (fileUrl) {
+ TString type = "REGULAR";
+ if (auto t = fileAttrs.FindPtr("type")) {
+ type = t->GetString();
+ }
+ TStringBuilder alias;
+ if (!prefix.empty()) {
+ alias << prefix << "/";
+ }
+ alias << file.first;
+ if (type == "REGULAR") {
+ if (!AddFileByUrlImpl(alias, TStringBuf(MakeHttps(fileUrl->GetString())), pos, ctx)) {
+ return false;
+ }
+ } else if (type == "DIRECTORY") {
+ queue.emplace_back(alias, fileUrl->GetString());
+ if (++count > Types.FolderSubDirsLimit) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Sandbox resource has too many subfolders. Limit is " << Types.FolderSubDirsLimit));
+ return false;
+ }
+ } else {
+ YQL_CLOG(WARN, ProviderConfig) << "Got unknown sandbox item type: " << type << ", name=" << alias;
+ }
+ }
+ }
+ }
+
+ return true;
+
+ }
+
+ bool SetWarningRule(const TPosition& pos, const TVector<TStringBuf>& args, TExprContext& ctx) {
+ if (args.size() != 2) {
+ ctx.AddError(TIssue(pos, TStringBuilder() << "Expected 2 arguments, but got " << args.size()));
+ return false;
+ }
+
TString codePattern = TString{args[0]};
TString action = TString{args[1]};
-
- TWarningRule rule;
- TString parseError;
- auto parseResult = TWarningRule::ParseFrom(codePattern, action, rule, parseError);
- switch (parseResult) {
- case TWarningRule::EParseResult::PARSE_OK:
- ctx.IssueManager.AddWarningRule(rule);
- break;
- case TWarningRule::EParseResult::PARSE_PATTERN_FAIL:
- case TWarningRule::EParseResult::PARSE_ACTION_FAIL:
- ctx.AddError(TIssue(pos, parseError));
- break;
- default:
- YQL_ENSURE(false, "Unknown parse result");
- }
-
- return parseResult == TWarningRule::EParseResult::PARSE_OK;
- }
-
- private:
+
+ TWarningRule rule;
+ TString parseError;
+ auto parseResult = TWarningRule::ParseFrom(codePattern, action, rule, parseError);
+ switch (parseResult) {
+ case TWarningRule::EParseResult::PARSE_OK:
+ ctx.IssueManager.AddWarningRule(rule);
+ break;
+ case TWarningRule::EParseResult::PARSE_PATTERN_FAIL:
+ case TWarningRule::EParseResult::PARSE_ACTION_FAIL:
+ ctx.AddError(TIssue(pos, parseError));
+ break;
+ default:
+ YQL_ENSURE(false, "Unknown parse result");
+ }
+
+ return parseResult == TWarningRule::EParseResult::PARSE_OK;
+ }
+
+ private:
TTypeAnnotationContext& Types;
TAutoPtr<IGraphTransformer> TypeAnnotationTransformer;
TAutoPtr<IGraphTransformer> ConfigurationTransformer;
TAutoPtr<IGraphTransformer> CallableExecutionTransformer;
- const TYqlCoreConfig* CoreConfig;
+ const TYqlCoreConfig* CoreConfig;
const TAllowSettingPolicy Policy;
};
}
diff --git a/ydb/library/yql/providers/config/yql_config_provider.h b/ydb/library/yql/providers/config/yql_config_provider.h
index ea1f47508a..3d3e608358 100644
--- a/ydb/library/yql/providers/config/yql_config_provider.h
+++ b/ydb/library/yql/providers/config/yql_config_provider.h
@@ -6,10 +6,10 @@
namespace NYql {
-class TGatewaysConfig;
-
+class TGatewaysConfig;
+
const TStringBuf ConfReadName = "ConfRead!";
-
+
using TAllowSettingPolicy = std::function<bool(TStringBuf settingName)>;
TIntrusivePtr<IDataProvider> CreateConfigProvider(TTypeAnnotationContext& types, const TGatewaysConfig* config,
diff --git a/ydb/library/yql/providers/dq/actors/actor_helpers.h b/ydb/library/yql/providers/dq/actors/actor_helpers.h
index b92603ab0f..b75a5ea676 100644
--- a/ydb/library/yql/providers/dq/actors/actor_helpers.h
+++ b/ydb/library/yql/providers/dq/actors/actor_helpers.h
@@ -12,7 +12,7 @@ namespace NYql {
enum EExecutorPoolType {
Main,
- FullResultWriter,
+ FullResultWriter,
TotalCount,
};
diff --git a/ydb/library/yql/providers/dq/actors/executer_actor.cpp b/ydb/library/yql/providers/dq/actors/executer_actor.cpp
index 91f1caee74..bf683da6f0 100644
--- a/ydb/library/yql/providers/dq/actors/executer_actor.cpp
+++ b/ydb/library/yql/providers/dq/actors/executer_actor.cpp
@@ -81,8 +81,8 @@ private:
HFunc(TEvQueryResponse, OnQueryResponse);
// execution timeout
cFunc(TEvents::TEvBootstrap::EventType, [this]() {
- YQL_LOG_CTX_SCOPE(TraceId);
- YQL_LOG(DEBUG) << "Execution timeout";
+ YQL_LOG_CTX_SCOPE(TraceId);
+ YQL_LOG(DEBUG) << "Execution timeout";
auto issue = TIssue("Execution timeout");
issue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR);
Issues.AddIssues({issue});
@@ -238,7 +238,7 @@ private:
void Finish(bool retriable, bool needFallback = false)
{
- YQL_LOG(DEBUG) << __FUNCTION__ << ", retriable=" << retriable << ", needFallback=" << needFallback;
+ YQL_LOG(DEBUG) << __FUNCTION__ << ", retriable=" << retriable << ", needFallback=" << needFallback;
if (Finished) {
YQL_LOG(WARN) << "Re-Finish IGNORED with Retriable=" << retriable << ", NeedFallback=" << needFallback;
} else {
@@ -256,7 +256,7 @@ private:
void OnFailure(TEvDqFailure::TPtr& ev, const NActors::TActorContext&) {
if (!Finished) {
YQL_LOG_CTX_SCOPE(TraceId);
- YQL_LOG(DEBUG) << __FUNCTION__;
+ YQL_LOG(DEBUG) << __FUNCTION__;
AddCounters(ev->Get()->Record);
bool retriable = ev->Get()->Record.GetRetriable();
bool fallback = ev->Get()->Record.GetNeedFallback();
@@ -265,13 +265,13 @@ private:
IssuesFromMessage(ev->Get()->Record.GetIssues(), issues);
Issues.AddIssues(issues);
}
- Finish(retriable, fallback);
+ Finish(retriable, fallback);
}
}
void OnGraphFinished(TEvGraphFinished::TPtr&, const NActors::TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceId);
- YQL_LOG(DEBUG) << __FUNCTION__;
+ YQL_LOG_CTX_SCOPE(TraceId);
+ YQL_LOG(DEBUG) << __FUNCTION__;
if (!Finished) {
try {
TFailureInjector::Reach("dq_fail_on_finish", [] { throw yexception() << "dq_fail_on_finish"; });
@@ -287,8 +287,8 @@ private:
// TBD: wait for PoisonTaken from CheckPointCoordinator before send TEvQueryResponse to PrinterId
void OnQueryResponse(TEvQueryResponse::TPtr& ev, const TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceId);
- YQL_LOG(DEBUG) << __FUNCTION__;
+ YQL_LOG_CTX_SCOPE(TraceId);
+ YQL_LOG(DEBUG) << __FUNCTION__;
Send(PrinterId, ev->Release().Release());
PassAway();
}
diff --git a/ydb/library/yql/providers/dq/actors/full_result_writer.cpp b/ydb/library/yql/providers/dq/actors/full_result_writer.cpp
index 927ac06043..26a74bd82a 100644
--- a/ydb/library/yql/providers/dq/actors/full_result_writer.cpp
+++ b/ydb/library/yql/providers/dq/actors/full_result_writer.cpp
@@ -1,18 +1,18 @@
-#include "proto_builder.h"
-#include "full_result_writer.h"
+#include "proto_builder.h"
+#include "full_result_writer.h"
#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
#include <ydb/library/yql/providers/dq/actors/events.h>
#include <ydb/library/yql/providers/dq/api/protos/service.pb.h>
#include <ydb/library/yql/core/issue/yql_issue.h>
-
+
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/failure_injector/failure_injector.h>
#include <ydb/library/yql/utils/yql_panic.h>
-#include <library/cpp/actors/core/actor.h>
-
+#include <library/cpp/actors/core/actor.h>
+
#include <util/generic/size_literals.h>
#include <util/system/env.h>
@@ -22,20 +22,20 @@ namespace NYql::NDqs {
using namespace NKikimr::NMiniKQL;
-class TFullResultWriterActor : public NActors::TActor<TFullResultWriterActor> {
+class TFullResultWriterActor : public NActors::TActor<TFullResultWriterActor> {
public:
static constexpr char ActorName[] = "YQL_DQ_FULL_RESULT_WRITER";
- explicit TFullResultWriterActor(const TString& traceId,
- const TString& resultType,
- THolder<IDqFullResultWriter>&& writer,
- const NActors::TActorId& aggregatorId)
- : NActors::TActor<TFullResultWriterActor>(&TFullResultWriterActor::Handler)
- , TraceID(traceId)
+ explicit TFullResultWriterActor(const TString& traceId,
+ const TString& resultType,
+ THolder<IDqFullResultWriter>&& writer,
+ const NActors::TActorId& aggregatorId)
+ : NActors::TActor<TFullResultWriterActor>(&TFullResultWriterActor::Handler)
+ , TraceID(traceId)
, ResultBuilder(MakeHolder<TProtoBuilder>(resultType, TVector<TString>()))
- , FullResultWriter(std::move(writer))
+ , FullResultWriter(std::move(writer))
, AggregatorID(aggregatorId)
- {
+ {
}
private:
@@ -46,23 +46,23 @@ private:
})
void PassAway() override {
- YQL_LOG_CTX_SCOPE(TraceID);
- YQL_LOG(DEBUG) << __FUNCTION__;
+ YQL_LOG_CTX_SCOPE(TraceID);
+ YQL_LOG(DEBUG) << __FUNCTION__;
try {
FullResultWriter->Abort();
} catch (...) {
YQL_LOG(WARN) << "FullResultWriter->Abort(): " << CurrentExceptionMessage();
}
- ResultBuilder.Reset();
+ ResultBuilder.Reset();
FullResultWriter.Reset();
Send(AggregatorID, MakeHolder<NActors::TEvents::TEvGone>());
- NActors::TActor<TThis>::PassAway();
+ NActors::TActor<TThis>::PassAway();
}
void OnStatusRequest(TEvFullResultWriterStatusRequest::TPtr&, const NActors::TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceID);
+ YQL_LOG_CTX_SCOPE(TraceID);
NDqProto::TFullResultWriterStatusResponse response;
response.SetBytesReceived(BytesReceived);
if (ErrorMessage) {
@@ -72,7 +72,7 @@ private:
}
void OnPullResponse(TEvPullDataResponse::TPtr& ev, const NActors::TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceID);
+ YQL_LOG_CTX_SCOPE(TraceID);
auto& response = ev->Get()->Record;
switch (response.GetResponseType()) {
@@ -89,26 +89,26 @@ private:
}
void Finish() {
- YQL_LOG(DEBUG) << __FUNCTION__;
+ YQL_LOG(DEBUG) << __FUNCTION__;
try {
- TFailureInjector::Reach("full_result_fail_on_finish", [] { throw yexception() << "full_result_fail_on_finish"; });
+ TFailureInjector::Reach("full_result_fail_on_finish", [] { throw yexception() << "full_result_fail_on_finish"; });
FullResultWriter->Finish();
- if (ErrorMessage) {
- TIssue issue(*ErrorMessage);
- issue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR);
- Send(AggregatorID, MakeHolder<TEvDqFailure>(issue, false, false).Release());
- } else {
- Send(AggregatorID, MakeHolder<TEvDqFailure>().Release());
- }
+ if (ErrorMessage) {
+ TIssue issue(*ErrorMessage);
+ issue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR);
+ Send(AggregatorID, MakeHolder<TEvDqFailure>(issue, false, false).Release());
+ } else {
+ Send(AggregatorID, MakeHolder<TEvDqFailure>().Release());
+ }
} catch (...) {
TIssue issue(CurrentExceptionMessage());
issue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR);
if (ErrorMessage) {
issue.AddSubIssue(MakeIntrusive<TIssue>(*ErrorMessage));
}
- Send(AggregatorID, MakeHolder<TEvDqFailure>(issue, false, false).Release());
+ Send(AggregatorID, MakeHolder<TEvDqFailure>(issue, false, false).Release());
}
- Send(SelfId(), MakeHolder<NActors::TEvents::TEvPoison>());
+ Send(SelfId(), MakeHolder<NActors::TEvents::TEvPoison>());
}
void Continue(NDqProto::TPullResponse& response) {
@@ -125,11 +125,11 @@ private:
}
try {
- TFailureInjector::Reach("full_result_fail_on_write", [] { throw yexception() << "full_result_fail_on_write"; });
- ResultBuilder->WriteData(data, [writer = FullResultWriter.Get()](const NUdf::TUnboxedValuePod& value) {
- writer->AddRow(value);
- return true;
- });
+ TFailureInjector::Reach("full_result_fail_on_write", [] { throw yexception() << "full_result_fail_on_write"; });
+ ResultBuilder->WriteData(data, [writer = FullResultWriter.Get()](const NUdf::TUnboxedValuePod& value) {
+ writer->AddRow(value);
+ return true;
+ });
} catch (...) {
ErrorMessage = CurrentExceptionMessage();
}
@@ -138,24 +138,24 @@ private:
YQL_LOG(DEBUG) << "An error occurred: " << *ErrorMessage;
}
}
-private:
- const TString TraceID;
- THolder<TProtoBuilder> ResultBuilder;
- THolder<IDqFullResultWriter> FullResultWriter;
- NActors::TActorId AggregatorID;
-
- ui64 BytesReceived{0};
- TMaybe<TString> ErrorMessage;
+private:
+ const TString TraceID;
+ THolder<TProtoBuilder> ResultBuilder;
+ THolder<IDqFullResultWriter> FullResultWriter;
+ NActors::TActorId AggregatorID;
+
+ ui64 BytesReceived{0};
+ TMaybe<TString> ErrorMessage;
};
THolder<NActors::IActor> MakeFullResultWriterActor(
- const TString& traceId,
- const TString& resultType,
- THolder<IDqFullResultWriter>&& writer,
- const NActors::TActorId& aggregatorId)
-{
- return MakeHolder<TFullResultWriterActor>(traceId, resultType, std::move(writer), aggregatorId);
+ const TString& traceId,
+ const TString& resultType,
+ THolder<IDqFullResultWriter>&& writer,
+ const NActors::TActorId& aggregatorId)
+{
+ return MakeHolder<TFullResultWriterActor>(traceId, resultType, std::move(writer), aggregatorId);
}
} // namespace NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/actors/full_result_writer.h b/ydb/library/yql/providers/dq/actors/full_result_writer.h
index 4d4345522a..92498051d9 100644
--- a/ydb/library/yql/providers/dq/actors/full_result_writer.h
+++ b/ydb/library/yql/providers/dq/actors/full_result_writer.h
@@ -6,9 +6,9 @@
namespace NYql::NDqs {
THolder<NActors::IActor> MakeFullResultWriterActor(
- const TString& traceId,
- const TString& resultType,
- THolder<IDqFullResultWriter>&& writer,
- const NActors::TActorId& aggregatorId);
+ const TString& traceId,
+ const TString& resultType,
+ THolder<IDqFullResultWriter>&& writer,
+ const NActors::TActorId& aggregatorId);
} // namespace NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.cpp b/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.cpp
index 3ff4fb1658..f15870b526 100644
--- a/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.cpp
+++ b/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.cpp
@@ -1,70 +1,70 @@
#include "graph_execution_events_actor.h"
-#include "actor_helpers.h"
-#include "full_result_writer.h"
-#include "events.h"
+#include "actor_helpers.h"
+#include "full_result_writer.h"
+#include "events.h"
-#include <library/cpp/actors/core/event_pb.h>
+#include <library/cpp/actors/core/event_pb.h>
-#include <util/generic/yexception.h>
-
-namespace NYql::NDqs {
+#include <util/generic/yexception.h>
+
+namespace NYql::NDqs {
class TGraphExecutionEventsActor : public TRichActor<TGraphExecutionEventsActor> {
public:
static constexpr const char ActorName[] = "YQL_DQ_GRAPH_EXECUTION_EVENTS_ACTOR";
- TGraphExecutionEventsActor(const TString& traceID, std::vector<IDqTaskPreprocessor::TPtr>&& taskPreprocessors)
+ TGraphExecutionEventsActor(const TString& traceID, std::vector<IDqTaskPreprocessor::TPtr>&& taskPreprocessors)
: TRichActor<TGraphExecutionEventsActor>(&TGraphExecutionEventsActor::Handler)
- , TraceID(traceID)
- , TaskPreprocessors(std::move(taskPreprocessors))
- {
- }
+ , TraceID(traceID)
+ , TaskPreprocessors(std::move(taskPreprocessors))
+ {
+ }
private:
- const TString TraceID;
- std::vector<IDqTaskPreprocessor::TPtr> TaskPreprocessors;
+ const TString TraceID;
+ std::vector<IDqTaskPreprocessor::TPtr> TaskPreprocessors;
STRICT_STFUNC(Handler, {
HFunc(NDqs::TEvGraphExecutionEvent, OnEvent);
cFunc(NActors::TEvents::TEvPoison::EventType, PassAway);
})
- void DoPassAway() override {
- YQL_LOG_CTX_SCOPE(TraceID);
- }
-
+ void DoPassAway() override {
+ YQL_LOG_CTX_SCOPE(TraceID);
+ }
+
void OnEvent(NDqs::TEvGraphExecutionEvent::TPtr& ev, const NActors::TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceID);
- YQL_LOG(DEBUG) << __FUNCTION__ << ' ' << NYql::NDqProto::EGraphExecutionEventType_Name(ev->Get()->Record.GetEventType());
+ YQL_LOG_CTX_SCOPE(TraceID);
+ YQL_LOG(DEBUG) << __FUNCTION__ << ' ' << NYql::NDqProto::EGraphExecutionEventType_Name(ev->Get()->Record.GetEventType());
try {
switch (ev->Get()->Record.GetEventType()) {
case NYql::NDqProto::EGraphExecutionEventType::START: {
NDqProto::TGraphExecutionEvent::TExecuteGraphDescriptor payload;
ev->Get()->Record.GetMessage().UnpackTo(&payload);
- OnStart(ev->Sender, payload);
- break;
- }
+ OnStart(ev->Sender, payload);
+ break;
+ }
case NYql::NDqProto::EGraphExecutionEventType::SUCCESS:
- OnSuccess(ev->Sender);
+ OnSuccess(ev->Sender);
break;
case NYql::NDqProto::EGraphExecutionEventType::FAIL:
- OnFail(ev->Sender);
- break;
- case NYql::NDqProto::EGraphExecutionEventType::FULL_RESULT: {
- NDqProto::TGraphExecutionEvent::TFullResultDescriptor payload;
- ev->Get()->Record.GetMessage().UnpackTo(&payload);
- OnFullResult(ev->Sender, payload);
+ OnFail(ev->Sender);
break;
- }
+ case NYql::NDqProto::EGraphExecutionEventType::FULL_RESULT: {
+ NDqProto::TGraphExecutionEvent::TFullResultDescriptor payload;
+ ev->Get()->Record.GetMessage().UnpackTo(&payload);
+ OnFullResult(ev->Sender, payload);
+ break;
+ }
default:
- Reply(ev->Sender);
+ Reply(ev->Sender);
break;
}
} catch (...) {
TString message = TStringBuilder() << "Error on TEvGraphExecutionEvent: " << CurrentExceptionMessage();
YQL_LOG(ERROR) << message;
- Reply(ev->Sender, message);
+ Reply(ev->Sender, message);
}
}
@@ -73,80 +73,80 @@ private:
return THashMap<TKey, TValue>(map.begin(), map.end());
}
- void OnStart(NActors::TActorId replyTo, const NDqProto::TGraphExecutionEvent::TExecuteGraphDescriptor& payload) {
- YQL_LOG(DEBUG) << __FUNCTION__;
+ void OnStart(NActors::TActorId replyTo, const NDqProto::TGraphExecutionEvent::TExecuteGraphDescriptor& payload) {
+ YQL_LOG(DEBUG) << __FUNCTION__;
const auto& secureParams = AsHashMap(payload.GetSecureParams().GetData());
- const auto& graphParams = AsHashMap(payload.GetGraphParams().GetData());
+ const auto& graphParams = AsHashMap(payload.GetGraphParams().GetData());
- NDqProto::TGraphExecutionEvent::TMap response;
+ NDqProto::TGraphExecutionEvent::TMap response;
- auto* addedParams = response.MutableData();
- for (const auto& preprocessor: TaskPreprocessors) {
- auto newParams = preprocessor->GetTaskParams(graphParams, secureParams);
- for (const auto& [k, v] : newParams) {
- addedParams->insert({k, v});
+ auto* addedParams = response.MutableData();
+ for (const auto& preprocessor: TaskPreprocessors) {
+ auto newParams = preprocessor->GetTaskParams(graphParams, secureParams);
+ for (const auto& [k, v] : newParams) {
+ addedParams->insert({k, v});
}
}
- Reply(replyTo, response);
+ Reply(replyTo, response);
}
- void Reply(NActors::TActorId replyTo, TString msg = "") const {
+ void Reply(NActors::TActorId replyTo, TString msg = "") const {
NYql::NDqProto::TGraphExecutionEvent sync;
sync.SetEventType(NDqProto::EGraphExecutionEventType::SYNC);
if (msg) {
sync.SetErrorMessage(msg);
}
- Send(replyTo, MakeHolder<NDqs::TEvGraphExecutionEvent>(sync));
+ Send(replyTo, MakeHolder<NDqs::TEvGraphExecutionEvent>(sync));
}
template <class TPayload>
- void Reply(NActors::TActorId replyTo, const TPayload& resp) const {
+ void Reply(NActors::TActorId replyTo, const TPayload& resp) const {
NYql::NDqProto::TGraphExecutionEvent sync;
sync.SetEventType(NDqProto::EGraphExecutionEventType::SYNC);
sync.MutableMessage()->PackFrom(resp);
- Send(replyTo, MakeHolder<NDqs::TEvGraphExecutionEvent>(sync));
- }
-
- void OnFail(NActors::TActorId replyTo) {
- YQL_LOG(DEBUG) << __FUNCTION__;
- for (const auto& preprocessor: TaskPreprocessors) {
- preprocessor->Finish(false);
- }
- Reply(replyTo);
+ Send(replyTo, MakeHolder<NDqs::TEvGraphExecutionEvent>(sync));
}
- void OnSuccess(NActors::TActorId replyTo) {
- YQL_LOG(DEBUG) << __FUNCTION__;
- for (const auto& preprocessor: TaskPreprocessors) {
- preprocessor->Finish(true);
+ void OnFail(NActors::TActorId replyTo) {
+ YQL_LOG(DEBUG) << __FUNCTION__;
+ for (const auto& preprocessor: TaskPreprocessors) {
+ preprocessor->Finish(false);
}
- Reply(replyTo);
+ Reply(replyTo);
}
- void OnFullResult(NActors::TActorId replyTo, const NDqProto::TGraphExecutionEvent::TFullResultDescriptor& payload) {
- YQL_LOG(DEBUG) << __FUNCTION__;
- THolder<IDqFullResultWriter> writer;
- for (const auto& preprocessor: TaskPreprocessors) {
- writer = preprocessor->CreateFullResultWriter();
- if (writer) {
- break;
- }
- }
- if (writer) {
- auto actor = MakeFullResultWriterActor(TraceID, payload.GetResultType(), std::move(writer), replyTo);
- auto fullResultWriterID = RegisterChild(actor.Release(), nullptr, EExecutorPoolType::FullResultWriter);
- NActorsProto::TActorId fullResultWriterProto;
- NActors::ActorIdToProto(fullResultWriterID, &fullResultWriterProto);
- Reply(replyTo, fullResultWriterProto);
- } else {
- Reply(replyTo, TString{"Failed to create full result writer"});
+ void OnSuccess(NActors::TActorId replyTo) {
+ YQL_LOG(DEBUG) << __FUNCTION__;
+ for (const auto& preprocessor: TaskPreprocessors) {
+ preprocessor->Finish(true);
}
+ Reply(replyTo);
}
+
+ void OnFullResult(NActors::TActorId replyTo, const NDqProto::TGraphExecutionEvent::TFullResultDescriptor& payload) {
+ YQL_LOG(DEBUG) << __FUNCTION__;
+ THolder<IDqFullResultWriter> writer;
+ for (const auto& preprocessor: TaskPreprocessors) {
+ writer = preprocessor->CreateFullResultWriter();
+ if (writer) {
+ break;
+ }
+ }
+ if (writer) {
+ auto actor = MakeFullResultWriterActor(TraceID, payload.GetResultType(), std::move(writer), replyTo);
+ auto fullResultWriterID = RegisterChild(actor.Release(), nullptr, EExecutorPoolType::FullResultWriter);
+ NActorsProto::TActorId fullResultWriterProto;
+ NActors::ActorIdToProto(fullResultWriterID, &fullResultWriterProto);
+ Reply(replyTo, fullResultWriterProto);
+ } else {
+ Reply(replyTo, TString{"Failed to create full result writer"});
+ }
+ }
};
-NActors::IActor* MakeGraphExecutionEventsActor(const TString& traceID, std::vector<IDqTaskPreprocessor::TPtr>&& taskPreprocessors) {
- return new TGraphExecutionEventsActor(traceID, std::move(taskPreprocessors));
+NActors::IActor* MakeGraphExecutionEventsActor(const TString& traceID, std::vector<IDqTaskPreprocessor::TPtr>&& taskPreprocessors) {
+ return new TGraphExecutionEventsActor(traceID, std::move(taskPreprocessors));
}
-} // namespace NYql::NDqs
+} // namespace NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.h b/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.h
index 73e6eba0fb..3c13b0508a 100644
--- a/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.h
+++ b/ydb/library/yql/providers/dq/actors/graph_execution_events_actor.h
@@ -3,8 +3,8 @@
#include <library/cpp/actors/core/actor.h>
#include <ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.h>
-namespace NYql::NDqs {
+namespace NYql::NDqs {
-NActors::IActor* MakeGraphExecutionEventsActor(const TString& traceID, std::vector<IDqTaskPreprocessor::TPtr>&& taskPreprocessors);
+NActors::IActor* MakeGraphExecutionEventsActor(const TString& traceID, std::vector<IDqTaskPreprocessor::TPtr>&& taskPreprocessors);
-} // namespace NYql::NDq
+} // namespace NYql::NDq
diff --git a/ydb/library/yql/providers/dq/actors/proto_builder.cpp b/ydb/library/yql/providers/dq/actors/proto_builder.cpp
index e4b16403c3..f10d738796 100644
--- a/ydb/library/yql/providers/dq/actors/proto_builder.cpp
+++ b/ydb/library/yql/providers/dq/actors/proto_builder.cpp
@@ -14,164 +14,164 @@
namespace NYql::NDqs {
-using namespace NKikimr::NMiniKQL;
+using namespace NKikimr::NMiniKQL;
-namespace {
+namespace {
-TVector<ui32> BuildColumnOrder(const TVector<TString>& columns, NKikimr::NMiniKQL::TType* resultType) {
- MKQL_ENSURE(resultType, "Incorrect result type");
- if (resultType->GetKind() != TType::EKind::Struct || columns.empty()) {
- return {};
+TVector<ui32> BuildColumnOrder(const TVector<TString>& columns, NKikimr::NMiniKQL::TType* resultType) {
+ MKQL_ENSURE(resultType, "Incorrect result type");
+ if (resultType->GetKind() != TType::EKind::Struct || columns.empty()) {
+ return {};
}
- TVector<ui32> columnOrder;
- THashMap<TString, ui32> column2id;
- auto structType = AS_TYPE(TStructType, resultType);
- for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
- const auto columnName = TString(structType->GetMemberName(i));
- column2id[columnName] = i;
+ TVector<ui32> columnOrder;
+ THashMap<TString, ui32> column2id;
+ auto structType = AS_TYPE(TStructType, resultType);
+ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
+ const auto columnName = TString(structType->GetMemberName(i));
+ column2id[columnName] = i;
}
- columnOrder.resize(columns.size());
+ columnOrder.resize(columns.size());
- int id = 0;
- for (const auto& columnName : columns) {
- columnOrder[id++] = column2id[columnName];
+ int id = 0;
+ for (const auto& columnName : columns) {
+ columnOrder[id++] = column2id[columnName];
}
- return columnOrder;
-}
-
-NDqProto::EDataTransportVersion GetTransportVersion(const NYql::NDqProto::TData& data) {
- switch (data.GetTransportVersion()) {
- case 10000:
- return NDqProto::EDataTransportVersion::DATA_TRANSPORT_YSON_1_0;
- case 20000:
- return NDqProto::EDataTransportVersion::DATA_TRANSPORT_UV_PICKLE_1_0;
- case 30000:
- return NDqProto::EDataTransportVersion::DATA_TRANSPORT_ARROW_1_0;
- default:
- break;
+ return columnOrder;
+}
+
+NDqProto::EDataTransportVersion GetTransportVersion(const NYql::NDqProto::TData& data) {
+ switch (data.GetTransportVersion()) {
+ case 10000:
+ return NDqProto::EDataTransportVersion::DATA_TRANSPORT_YSON_1_0;
+ case 20000:
+ return NDqProto::EDataTransportVersion::DATA_TRANSPORT_UV_PICKLE_1_0;
+ case 30000:
+ return NDqProto::EDataTransportVersion::DATA_TRANSPORT_ARROW_1_0;
+ default:
+ break;
}
- return NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED;
-}
-
-} // unnamed
-
-TProtoBuilder::TProtoBuilder(const TString& type, const TVector<TString>& columns)
- : Alloc()
- , TypeEnv(Alloc)
- , ResultType(reinterpret_cast<TType*>(DeserializeNode(type, TypeEnv)))
- , ColumnOrder(BuildColumnOrder(columns, ResultType))
-{
- Alloc.Release();
-}
-
-TProtoBuilder::~TProtoBuilder() {
- Alloc.Acquire();
-}
-
-bool TProtoBuilder::CanBuildResultSet() const {
- return ResultType->GetKind() == TType::EKind::Struct;
-}
-
-TString TProtoBuilder::BuildYson(const TVector<NYql::NDqProto::TData>& rows, ui64 maxBytesLimit) {
- ui64 size = 0;
- TStringStream out;
- NYson::TYsonWriter writer((IOutputStream*)&out);
- writer.OnBeginList();
-
- auto full = WriteData(rows, [&](const NYql::NUdf::TUnboxedValuePod& value) {
- auto rowYson = NCommon::WriteYsonValue(value, ResultType, ColumnOrder.empty() ? nullptr : &ColumnOrder);
- writer.OnListItem();
- writer.OnRaw(rowYson);
- size += rowYson.size();
- return size <= maxBytesLimit;
- });
-
- if (!full) {
- ythrow yexception() << "Too big yson result size: " << size << " > " << maxBytesLimit;
+ return NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED;
+}
+
+} // unnamed
+
+TProtoBuilder::TProtoBuilder(const TString& type, const TVector<TString>& columns)
+ : Alloc()
+ , TypeEnv(Alloc)
+ , ResultType(reinterpret_cast<TType*>(DeserializeNode(type, TypeEnv)))
+ , ColumnOrder(BuildColumnOrder(columns, ResultType))
+{
+ Alloc.Release();
+}
+
+TProtoBuilder::~TProtoBuilder() {
+ Alloc.Acquire();
+}
+
+bool TProtoBuilder::CanBuildResultSet() const {
+ return ResultType->GetKind() == TType::EKind::Struct;
+}
+
+TString TProtoBuilder::BuildYson(const TVector<NYql::NDqProto::TData>& rows, ui64 maxBytesLimit) {
+ ui64 size = 0;
+ TStringStream out;
+ NYson::TYsonWriter writer((IOutputStream*)&out);
+ writer.OnBeginList();
+
+ auto full = WriteData(rows, [&](const NYql::NUdf::TUnboxedValuePod& value) {
+ auto rowYson = NCommon::WriteYsonValue(value, ResultType, ColumnOrder.empty() ? nullptr : &ColumnOrder);
+ writer.OnListItem();
+ writer.OnRaw(rowYson);
+ size += rowYson.size();
+ return size <= maxBytesLimit;
+ });
+
+ if (!full) {
+ ythrow yexception() << "Too big yson result size: " << size << " > " << maxBytesLimit;
}
- writer.OnEndList();
- return out.Str();
-}
-
-bool TProtoBuilder::WriteYsonData(const NYql::NDqProto::TData& data, const std::function<bool(const TString& rawYson)>& func) {
- return WriteData(data, [&](const NYql::NUdf::TUnboxedValuePod& value) {
- auto rowYson = NCommon::WriteYsonValue(value, ResultType, ColumnOrder.empty() ? nullptr : &ColumnOrder);
- return func(rowYson);
- });
-}
-
-bool TProtoBuilder::WriteData(const NDqProto::TData& data, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func) {
- TGuard<TScopedAlloc> allocGuard(Alloc);
-
- TMemoryUsageInfo memInfo("ProtoBuilder");
- THolderFactory holderFactory(Alloc.Ref(), memInfo);
- NDqProto::EDataTransportVersion transportVersion = GetTransportVersion(data);
- NDq::TDqDataSerializer dataSerializer(TypeEnv, holderFactory, transportVersion);
-
- TUnboxedValueVector buffer;
- dataSerializer.Deserialize(data, ResultType, buffer);
-
- for (const auto& item : buffer) {
- if (!func(item)) {
- return false;
+ writer.OnEndList();
+ return out.Str();
+}
+
+bool TProtoBuilder::WriteYsonData(const NYql::NDqProto::TData& data, const std::function<bool(const TString& rawYson)>& func) {
+ return WriteData(data, [&](const NYql::NUdf::TUnboxedValuePod& value) {
+ auto rowYson = NCommon::WriteYsonValue(value, ResultType, ColumnOrder.empty() ? nullptr : &ColumnOrder);
+ return func(rowYson);
+ });
+}
+
+bool TProtoBuilder::WriteData(const NDqProto::TData& data, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func) {
+ TGuard<TScopedAlloc> allocGuard(Alloc);
+
+ TMemoryUsageInfo memInfo("ProtoBuilder");
+ THolderFactory holderFactory(Alloc.Ref(), memInfo);
+ NDqProto::EDataTransportVersion transportVersion = GetTransportVersion(data);
+ NDq::TDqDataSerializer dataSerializer(TypeEnv, holderFactory, transportVersion);
+
+ TUnboxedValueVector buffer;
+ dataSerializer.Deserialize(data, ResultType, buffer);
+
+ for (const auto& item : buffer) {
+ if (!func(item)) {
+ return false;
}
}
- return true;
-}
-
-bool TProtoBuilder::WriteData(const TVector<NDqProto::TData>& rows, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func) {
- TGuard<TScopedAlloc> allocGuard(Alloc);
-
- TMemoryUsageInfo memInfo("ProtoBuilder");
- THolderFactory holderFactory(Alloc.Ref(), memInfo);
- const auto transportVersion = rows.empty() ? NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED : GetTransportVersion(rows.front());
- NDq::TDqDataSerializer dataSerializer(TypeEnv, holderFactory, transportVersion);
-
- for (const auto& part : rows) {
- TUnboxedValueVector buffer;
- dataSerializer.Deserialize(part, ResultType, buffer);
- for (const auto& item : buffer) {
- if (!func(item)) {
- return false;
+ return true;
+}
+
+bool TProtoBuilder::WriteData(const TVector<NDqProto::TData>& rows, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func) {
+ TGuard<TScopedAlloc> allocGuard(Alloc);
+
+ TMemoryUsageInfo memInfo("ProtoBuilder");
+ THolderFactory holderFactory(Alloc.Ref(), memInfo);
+ const auto transportVersion = rows.empty() ? NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED : GetTransportVersion(rows.front());
+ NDq::TDqDataSerializer dataSerializer(TypeEnv, holderFactory, transportVersion);
+
+ for (const auto& part : rows) {
+ TUnboxedValueVector buffer;
+ dataSerializer.Deserialize(part, ResultType, buffer);
+ for (const auto& item : buffer) {
+ if (!func(item)) {
+ return false;
}
}
}
- return true;
-}
-
-Ydb::ResultSet TProtoBuilder::BuildResultSet(const TVector<NYql::NDqProto::TData>& data) {
- Ydb::ResultSet resultSet;
- auto structType = AS_TYPE(TStructType, ResultType);
- MKQL_ENSURE(structType, "Result is not a struct");
- for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
- auto& column = *resultSet.add_columns();
- const ui32 memberIndex = ColumnOrder.empty() ? i : ColumnOrder[i];
- column.set_name(TString(structType->GetMemberName(memberIndex)));
- ExportTypeToProto(structType->GetMemberType(memberIndex), *column.mutable_type());
+ return true;
+}
+
+Ydb::ResultSet TProtoBuilder::BuildResultSet(const TVector<NYql::NDqProto::TData>& data) {
+ Ydb::ResultSet resultSet;
+ auto structType = AS_TYPE(TStructType, ResultType);
+ MKQL_ENSURE(structType, "Result is not a struct");
+ for (ui32 i = 0; i < structType->GetMembersCount(); ++i) {
+ auto& column = *resultSet.add_columns();
+ const ui32 memberIndex = ColumnOrder.empty() ? i : ColumnOrder[i];
+ column.set_name(TString(structType->GetMemberName(memberIndex)));
+ ExportTypeToProto(structType->GetMemberType(memberIndex), *column.mutable_type());
}
- WriteData(data, [&](const NYql::NUdf::TUnboxedValuePod& value) {
- ExportValueToProto(ResultType, value, *resultSet.add_rows(), &ColumnOrder);
- return true;
- });
-
- return resultSet;
-}
-
-TString TProtoBuilder::GetSerializedType() const {
- auto result = SerializeNode(ResultType, TypeEnv);
- return result;
+ WriteData(data, [&](const NYql::NUdf::TUnboxedValuePod& value) {
+ ExportValueToProto(ResultType, value, *resultSet.add_rows(), &ColumnOrder);
+ return true;
+ });
+
+ return resultSet;
}
-
-TString TProtoBuilder::AllocDebugInfo() {
- TGuard<TScopedAlloc> allocGuard(Alloc);
- return TStringBuilder{} << "Used: " << Alloc.GetUsed() << '\n'
- << "Peak used: " << Alloc.GetPeakUsed() << '\n'
- << "Allocated: " << Alloc.GetAllocated() << '\n'
- << "Peak allocated: " << Alloc.GetPeakAllocated() << '\n'
- << "Limit: " << Alloc.GetLimit();
-}
-
-} // NYql::NDqs
+
+TString TProtoBuilder::GetSerializedType() const {
+ auto result = SerializeNode(ResultType, TypeEnv);
+ return result;
+}
+
+TString TProtoBuilder::AllocDebugInfo() {
+ TGuard<TScopedAlloc> allocGuard(Alloc);
+ return TStringBuilder{} << "Used: " << Alloc.GetUsed() << '\n'
+ << "Peak used: " << Alloc.GetPeakUsed() << '\n'
+ << "Allocated: " << Alloc.GetAllocated() << '\n'
+ << "Peak allocated: " << Alloc.GetPeakAllocated() << '\n'
+ << "Limit: " << Alloc.GetLimit();
+}
+
+} // NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/actors/proto_builder.h b/ydb/library/yql/providers/dq/actors/proto_builder.h
index 8ba05c8c3d..a1dfa8bdcb 100644
--- a/ydb/library/yql/providers/dq/actors/proto_builder.h
+++ b/ydb/library/yql/providers/dq/actors/proto_builder.h
@@ -1,40 +1,40 @@
#pragma once
#include <ydb/public/api/protos/ydb_value.pb.h>
-
+
#include <ydb/library/yql/minikql/mkql_alloc.h>
#include <ydb/library/yql/minikql/mkql_node.h>
-namespace NYql {
- class IDqFullResultWriter;
-}
-
-namespace NYql::NDqProto {
- class TData;
-}
+namespace NYql {
+ class IDqFullResultWriter;
+}
+namespace NYql::NDqProto {
+ class TData;
+}
+
namespace NYql::NDqs {
-class TProtoBuilder {
+class TProtoBuilder {
public:
TProtoBuilder(const TString& type, const TVector<TString>& columns);
- ~TProtoBuilder();
+ ~TProtoBuilder();
bool CanBuildResultSet() const;
Ydb::ResultSet BuildResultSet(const TVector<NYql::NDqProto::TData>& data);
TString BuildYson(const TVector<NYql::NDqProto::TData>& data, ui64 maxBytesLimit = std::numeric_limits<ui64>::max());
- bool WriteYsonData(const NYql::NDqProto::TData& data, const std::function<bool(const TString& rawYson)>& func);
- bool WriteData(const NYql::NDqProto::TData& data, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func);
- bool WriteData(const TVector<NYql::NDqProto::TData>& data, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func);
- TString GetSerializedType() const;
- TString AllocDebugInfo();
-
-private:
- NKikimr::NMiniKQL::TScopedAlloc Alloc;
- NKikimr::NMiniKQL::TTypeEnvironment TypeEnv;
- NKikimr::NMiniKQL::TType* ResultType;
+ bool WriteYsonData(const NYql::NDqProto::TData& data, const std::function<bool(const TString& rawYson)>& func);
+ bool WriteData(const NYql::NDqProto::TData& data, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func);
+ bool WriteData(const TVector<NYql::NDqProto::TData>& data, const std::function<bool(const NYql::NUdf::TUnboxedValuePod& value)>& func);
+ TString GetSerializedType() const;
+ TString AllocDebugInfo();
+
+private:
+ NKikimr::NMiniKQL::TScopedAlloc Alloc;
+ NKikimr::NMiniKQL::TTypeEnvironment TypeEnv;
+ NKikimr::NMiniKQL::TType* ResultType;
const TVector<ui32> ColumnOrder;
-};
+};
-} // NYql::NDqs
+} // NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/actors/result_aggregator.cpp b/ydb/library/yql/providers/dq/actors/result_aggregator.cpp
index 59ae95c3af..6845937475 100644
--- a/ydb/library/yql/providers/dq/actors/result_aggregator.cpp
+++ b/ydb/library/yql/providers/dq/actors/result_aggregator.cpp
@@ -1,7 +1,7 @@
#include "result_aggregator.h"
#include "result_receiver.h"
-#include "proto_builder.h"
-#include "full_result_writer.h"
+#include "proto_builder.h"
+#include "full_result_writer.h"
#include <ydb/library/yql/providers/dq/actors/actor_helpers.h>
#include <ydb/library/yql/providers/dq/actors/events.h>
@@ -15,7 +15,7 @@
#include <ydb/library/yql/public/issue/yql_issue_message.h>
#include <ydb/library/yql/sql/sql.h>
-#include <ydb/library/yql/utils/failure_injector/failure_injector.h>
+#include <ydb/library/yql/utils/failure_injector/failure_injector.h>
#include <ydb/library/yql/utils/actor_log/log.h>
#include <ydb/library/yql/utils/log/log.h>
@@ -27,526 +27,526 @@
#include <library/cpp/actors/core/hfunc.h>
#include <library/cpp/actors/core/scheduler_basic.h>
#include <library/cpp/threading/future/future.h>
-#include <library/cpp/yson/writer.h>
+#include <library/cpp/yson/writer.h>
#include <util/generic/ptr.h>
#include <util/string/split.h>
#include <util/system/types.h>
-#include <util/stream/holder.h>
-#include <util/stream/str.h>
-#include <util/stream/length.h>
+#include <util/stream/holder.h>
+#include <util/stream/str.h>
+#include <util/stream/length.h>
-namespace NYql::NDqs::NExecutionHelpers {
+namespace NYql::NDqs::NExecutionHelpers {
-using namespace NYql;
-using namespace NYql::NDqProto;
-using namespace NYql::NNodes;
+using namespace NYql;
+using namespace NYql::NDqProto;
+using namespace NYql::NNodes;
-using namespace NKikimr::NMiniKQL;
+using namespace NKikimr::NMiniKQL;
-using namespace Yql::DqsProto;
+using namespace Yql::DqsProto;
-using namespace NYql::NDq;
-using namespace NYql::NDqs;
-using namespace NYql::NDqProto;
-using namespace NActors;
+using namespace NYql::NDq;
+using namespace NYql::NDqs;
+using namespace NYql::NDqProto;
+using namespace NActors;
-namespace {
+namespace {
-class TResultAggregator: public TSynchronizableRichActor<TResultAggregator>, NYql::TCounters {
- static constexpr ui32 MAX_RESULT_BATCH = 2048;
+class TResultAggregator: public TSynchronizableRichActor<TResultAggregator>, NYql::TCounters {
+ static constexpr ui32 MAX_RESULT_BATCH = 2048;
-public:
+public:
static constexpr char ActorName[] = "YQL_DQ_RESULT_AGGREGATOR";
- explicit TResultAggregator(const TVector<TString>& columns, const NActors::TActorId& executerId, const TString& traceId,
- const TDqConfiguration::TPtr& settings, const TString& resultType, NActors::TActorId graphExecutionEventsId, bool discard)
- : TSynchronizableRichActor<TResultAggregator>(&TResultAggregator::Handler)
- , ExecuterID(executerId)
- , GraphExecutionEventsId(graphExecutionEventsId)
- , Discard(discard)
- , TraceId(traceId)
- , Settings(settings)
+ explicit TResultAggregator(const TVector<TString>& columns, const NActors::TActorId& executerId, const TString& traceId,
+ const TDqConfiguration::TPtr& settings, const TString& resultType, NActors::TActorId graphExecutionEventsId, bool discard)
+ : TSynchronizableRichActor<TResultAggregator>(&TResultAggregator::Handler)
+ , ExecuterID(executerId)
+ , GraphExecutionEventsId(graphExecutionEventsId)
+ , Discard(discard)
+ , TraceId(traceId)
+ , Settings(settings)
, ResultBuilder(MakeHolder<TProtoBuilder>(resultType, columns))
- , ResultYsonOut(new THoldingStream<TCountingOutput>(MakeHolder<TStringOutput>(ResultYson)))
- , ResultYsonWriter(MakeHolder<NYson::TYsonWriter>(ResultYsonOut.Get(), NYson::EYsonFormat::Binary, ::NYson::EYsonType::Node, true))
- {
- ResultYsonWriter->OnBeginList();
- if (Settings) {
- PullRequestTimeout = TDuration::MilliSeconds(settings->PullRequestTimeoutMs.Get().GetOrElse(0));
- PingTimeout = TDuration::MilliSeconds(settings->PingTimeoutMs.Get().GetOrElse(0));
- PingPeriod = Max(PingTimeout/4, TDuration::MilliSeconds(1000));
-
- SizeLimit = Settings->_AllResultsBytesLimit.Get().GetOrElse(64000000);
- YQL_LOG(DEBUG) << "_AllResultsBytesLimit = " << SizeLimit;
-
- if (Settings->_RowsLimitPerWrite.Get()) {
- YQL_LOG(DEBUG) << "_RowsLimitPerWrite = " << *Settings->_RowsLimitPerWrite.Get();
- RowsLimit = Settings->_RowsLimitPerWrite.Get();
- }
- }
- }
-
-private:
+ , ResultYsonOut(new THoldingStream<TCountingOutput>(MakeHolder<TStringOutput>(ResultYson)))
+ , ResultYsonWriter(MakeHolder<NYson::TYsonWriter>(ResultYsonOut.Get(), NYson::EYsonFormat::Binary, ::NYson::EYsonType::Node, true))
+ {
+ ResultYsonWriter->OnBeginList();
+ if (Settings) {
+ PullRequestTimeout = TDuration::MilliSeconds(settings->PullRequestTimeoutMs.Get().GetOrElse(0));
+ PingTimeout = TDuration::MilliSeconds(settings->PingTimeoutMs.Get().GetOrElse(0));
+ PingPeriod = Max(PingTimeout/4, TDuration::MilliSeconds(1000));
+
+ SizeLimit = Settings->_AllResultsBytesLimit.Get().GetOrElse(64000000);
+ YQL_LOG(DEBUG) << "_AllResultsBytesLimit = " << SizeLimit;
+
+ if (Settings->_RowsLimitPerWrite.Get()) {
+ YQL_LOG(DEBUG) << "_RowsLimitPerWrite = " << *Settings->_RowsLimitPerWrite.Get();
+ RowsLimit = Settings->_RowsLimitPerWrite.Get();
+ }
+ }
+ }
+
+private:
#define HANDLER_STUB(TEvType) \
- cFunc(TEvType::EventType, [this]() { \
- YQL_LOG_CTX_SCOPE(TraceId); \
- YQL_LOG(DEBUG) << "Unexpected event " << ( #TEvType ); \
- })
-
- STRICT_STFUNC(Handler, {
- HFunc(TEvPullResult, OnPullResult);
- HFunc(TEvReadyState, OnReadyState);
- HFunc(TEvPullDataResponse, OnPullResponse);
- HFunc(TEvPingResponse, OnPingResponse);
- HFunc(TEvQueryResponse, OnQueryResult);
- HFunc(TEvDqFailure, OnFullResultWriterResponse);
- cFunc(TEvents::TEvPoison::EventType, PassAway)
- hFunc(TEvents::TEvUndelivered, [this] (TEvents::TEvUndelivered::TPtr& ev) {
- YQL_LOG_CTX_SCOPE(TraceId);
- TString message = "Undelivered from " + ToString(ev->Sender) + " to " + ToString(SelfId())
- + " reason: " + ToString(ev->Get()->Reason) + " sourceType: " + ToString(ev->Get()->SourceType >> 16)
- + "." + ToString(ev->Get()->SourceType & 0xFFFF);
- OnError(message, true, true);
- })
- cFunc(TEvents::TEvWakeup::EventType, OnWakeup)
- cFunc(TEvents::TEvGone::EventType, OnFullResultWriterShutdown)
- })
-
-
- STRICT_STFUNC(ShutdownHandler, {
- HFunc(TEvents::TEvGone, OnShutdownQueryResult);
- HANDLER_STUB(TEvPullResult)
- HANDLER_STUB(TEvReadyState)
- HANDLER_STUB(TEvPullDataResponse)
- HANDLER_STUB(TEvPingResponse)
- HANDLER_STUB(TEvQueryResponse)
- HANDLER_STUB(TEvDqFailure)
- cFunc(TEvents::TEvPoison::EventType, PassAway);
- HANDLER_STUB(TEvents::TEvUndelivered)
- HANDLER_STUB(TEvents::TEvWakeup)
- })
-
- void DoPassAway() override {
- YQL_LOG_CTX_SCOPE(TraceId);
- YQL_LOG(DEBUG) << __FUNCTION__;
- }
-
- void OnFullResultWriterShutdown() {
- YQL_LOG_CTX_SCOPE(TraceId);
- YQL_LOG(DEBUG) << "Got TEvGone";
-
- FullResultWriterID = {};
- }
-
- void OnWakeup() {
- YQL_LOG_CTX_SCOPE(TraceId);
- YQL_LOG(DEBUG) << __FUNCTION__;
- auto now = TInstant::Now();
- if (PullRequestTimeout && now - PullRequestStartTime > PullRequestTimeout) {
- OnError("Timeout " + ToString(SourceID.NodeId()), true, true);
- }
-
- if (PingTimeout && now - PingStartTime > PingTimeout) {
- OnError("PingTimeout " + ToString(SourceID.NodeId()), true, true);
- }
-
- if (!PingRequested) {
- PingRequested = true;
- PingStartTime = now;
- Send(SourceID, MakeHolder<TEvPingRequest>(), IEventHandle::FlagTrackDelivery);
- }
-
- TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way());
- Schedule(PingPeriod, new TEvents::TEvWakeup(), TimerCookieHolder.Get());
- }
-
- void OnReadyState(TEvReadyState::TPtr& ev, const TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceId);
- AddCounters(ev->Get()->Record);
-
- SourceID = NActors::ActorIdFromProto(ev->Get()->Record.GetSourceId());
- Send(SelfId(), MakeHolder<TEvPullResult>());
-
- PingStartTime = PullRequestStartTime = TInstant::Now();
- TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way());
- Schedule(PingPeriod, new TEvents::TEvWakeup(), TimerCookieHolder.Get());
-
- AddCriticalEventType(TEvents::TEvWakeup::EventType);
- AddCriticalEventType(TEvPingResponse::EventType);
- }
-
- void OnPullResult(TEvPullResult::TPtr&, const TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceId);
- PullRequestStartTime = TInstant::Now();
- Send(SourceID, MakeHolder<TEvPullDataRequest>(MAX_RESULT_BATCH), IEventHandle::FlagTrackDelivery);
- }
-
- void OnError(const TString& message, bool retriable, bool needFallback) {
- YQL_LOG(ERROR) << "OnError " << message;
- auto issueCode = needFallback
- ? TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR
- : TIssuesIds::DQ_GATEWAY_ERROR;
- auto req = MakeHolder<TEvDqFailure>(TIssue(message).SetCode(issueCode, TSeverityIds::S_ERROR), retriable, needFallback);
- FlushCounters(req->Record);
- Send(ExecuterID, req.Release());
- }
-
- void OnPingResponse(TEvPingResponse::TPtr&, const TActorContext&) {
- PingRequested = false;
- }
-
- void OnPullResponse(TEvPullDataResponse::TPtr& ev, const TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceId);
-
- if (FinishCalled) {
- // finalization has been begun, actor will not kill himself anymore, should ignore responses instead
- return;
- }
-
- auto& response = ev->Get()->Record;
-
- AddCounters(response);
-
- switch (response.GetResponseType()) {
- case NYql::NDqProto::CONTINUE: {
- Send(SelfId(), MakeHolder<TEvPullResult>());
- } break;
- case NYql::NDqProto::FINISH:
- Finish(Truncated);
- return;
- case NYql::NDqProto::YIELD:
- Schedule(TDuration::MilliSeconds(10), new TEvPullResult());
- return;
- case NYql::NDqProto::ERROR: {
- OnError(ev->Get()->Record.GetErrorMessage(), false, true);
- break;
- }
- case NYql::NDqProto::UNKNOWN:
- [[fallthrough]];
- case NYql::NDqProto::EPullResponseType_INT_MIN_SENTINEL_DO_NOT_USE_:
- [[fallthrough]];
- case NYql::NDqProto::EPullResponseType_INT_MAX_SENTINEL_DO_NOT_USE_:
- YQL_ENSURE(false, "Unknown pull result");
- break;
- }
-
- if (!Discard) {
- auto fullResultTableEnabled = Settings->EnableFullResultWrite.Get().GetOrElse(false);
-
- if (fullResultTableEnabled && Truncated) {
- WriteToFullResultTable(new NDqProto::TData(std::move(*response.MutableData())));
- } else {
- DataParts.emplace_back(std::move(*response.MutableData()));
-
- bool full = true;
- bool exceedRows = false;
- try {
- full = ResultBuilder->WriteYsonData(DataParts.back(), [this, &exceedRows](const TString& rawYson) {
- if (RowsLimit && Rows + 1 > *RowsLimit) {
- exceedRows = true;
- return false;
- } else if (ResultYsonOut->Counter() + rawYson.size() > SizeLimit) {
- return false;
- }
- ResultYsonWriter->OnListItem();
- ResultYsonWriter->OnRaw(rawYson);
- ++Rows;
- return true;
- });
- } catch (...) {
- OnError(CurrentExceptionMessage(), false, false);
- return;
+ cFunc(TEvType::EventType, [this]() { \
+ YQL_LOG_CTX_SCOPE(TraceId); \
+ YQL_LOG(DEBUG) << "Unexpected event " << ( #TEvType ); \
+ })
+
+ STRICT_STFUNC(Handler, {
+ HFunc(TEvPullResult, OnPullResult);
+ HFunc(TEvReadyState, OnReadyState);
+ HFunc(TEvPullDataResponse, OnPullResponse);
+ HFunc(TEvPingResponse, OnPingResponse);
+ HFunc(TEvQueryResponse, OnQueryResult);
+ HFunc(TEvDqFailure, OnFullResultWriterResponse);
+ cFunc(TEvents::TEvPoison::EventType, PassAway)
+ hFunc(TEvents::TEvUndelivered, [this] (TEvents::TEvUndelivered::TPtr& ev) {
+ YQL_LOG_CTX_SCOPE(TraceId);
+ TString message = "Undelivered from " + ToString(ev->Sender) + " to " + ToString(SelfId())
+ + " reason: " + ToString(ev->Get()->Reason) + " sourceType: " + ToString(ev->Get()->SourceType >> 16)
+ + "." + ToString(ev->Get()->SourceType & 0xFFFF);
+ OnError(message, true, true);
+ })
+ cFunc(TEvents::TEvWakeup::EventType, OnWakeup)
+ cFunc(TEvents::TEvGone::EventType, OnFullResultWriterShutdown)
+ })
+
+
+ STRICT_STFUNC(ShutdownHandler, {
+ HFunc(TEvents::TEvGone, OnShutdownQueryResult);
+ HANDLER_STUB(TEvPullResult)
+ HANDLER_STUB(TEvReadyState)
+ HANDLER_STUB(TEvPullDataResponse)
+ HANDLER_STUB(TEvPingResponse)
+ HANDLER_STUB(TEvQueryResponse)
+ HANDLER_STUB(TEvDqFailure)
+ cFunc(TEvents::TEvPoison::EventType, PassAway);
+ HANDLER_STUB(TEvents::TEvUndelivered)
+ HANDLER_STUB(TEvents::TEvWakeup)
+ })
+
+ void DoPassAway() override {
+ YQL_LOG_CTX_SCOPE(TraceId);
+ YQL_LOG(DEBUG) << __FUNCTION__;
+ }
+
+ void OnFullResultWriterShutdown() {
+ YQL_LOG_CTX_SCOPE(TraceId);
+ YQL_LOG(DEBUG) << "Got TEvGone";
+
+ FullResultWriterID = {};
+ }
+
+ void OnWakeup() {
+ YQL_LOG_CTX_SCOPE(TraceId);
+ YQL_LOG(DEBUG) << __FUNCTION__;
+ auto now = TInstant::Now();
+ if (PullRequestTimeout && now - PullRequestStartTime > PullRequestTimeout) {
+ OnError("Timeout " + ToString(SourceID.NodeId()), true, true);
+ }
+
+ if (PingTimeout && now - PingStartTime > PingTimeout) {
+ OnError("PingTimeout " + ToString(SourceID.NodeId()), true, true);
+ }
+
+ if (!PingRequested) {
+ PingRequested = true;
+ PingStartTime = now;
+ Send(SourceID, MakeHolder<TEvPingRequest>(), IEventHandle::FlagTrackDelivery);
+ }
+
+ TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way());
+ Schedule(PingPeriod, new TEvents::TEvWakeup(), TimerCookieHolder.Get());
+ }
+
+ void OnReadyState(TEvReadyState::TPtr& ev, const TActorContext&) {
+ YQL_LOG_CTX_SCOPE(TraceId);
+ AddCounters(ev->Get()->Record);
+
+ SourceID = NActors::ActorIdFromProto(ev->Get()->Record.GetSourceId());
+ Send(SelfId(), MakeHolder<TEvPullResult>());
+
+ PingStartTime = PullRequestStartTime = TInstant::Now();
+ TimerCookieHolder.Reset(NActors::ISchedulerCookie::Make2Way());
+ Schedule(PingPeriod, new TEvents::TEvWakeup(), TimerCookieHolder.Get());
+
+ AddCriticalEventType(TEvents::TEvWakeup::EventType);
+ AddCriticalEventType(TEvPingResponse::EventType);
+ }
+
+ void OnPullResult(TEvPullResult::TPtr&, const TActorContext&) {
+ YQL_LOG_CTX_SCOPE(TraceId);
+ PullRequestStartTime = TInstant::Now();
+ Send(SourceID, MakeHolder<TEvPullDataRequest>(MAX_RESULT_BATCH), IEventHandle::FlagTrackDelivery);
+ }
+
+ void OnError(const TString& message, bool retriable, bool needFallback) {
+ YQL_LOG(ERROR) << "OnError " << message;
+ auto issueCode = needFallback
+ ? TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR
+ : TIssuesIds::DQ_GATEWAY_ERROR;
+ auto req = MakeHolder<TEvDqFailure>(TIssue(message).SetCode(issueCode, TSeverityIds::S_ERROR), retriable, needFallback);
+ FlushCounters(req->Record);
+ Send(ExecuterID, req.Release());
+ }
+
+ void OnPingResponse(TEvPingResponse::TPtr&, const TActorContext&) {
+ PingRequested = false;
+ }
+
+ void OnPullResponse(TEvPullDataResponse::TPtr& ev, const TActorContext&) {
+ YQL_LOG_CTX_SCOPE(TraceId);
+
+ if (FinishCalled) {
+ // finalization has been begun, actor will not kill himself anymore, should ignore responses instead
+ return;
+ }
+
+ auto& response = ev->Get()->Record;
+
+ AddCounters(response);
+
+ switch (response.GetResponseType()) {
+ case NYql::NDqProto::CONTINUE: {
+ Send(SelfId(), MakeHolder<TEvPullResult>());
+ } break;
+ case NYql::NDqProto::FINISH:
+ Finish(Truncated);
+ return;
+ case NYql::NDqProto::YIELD:
+ Schedule(TDuration::MilliSeconds(10), new TEvPullResult());
+ return;
+ case NYql::NDqProto::ERROR: {
+ OnError(ev->Get()->Record.GetErrorMessage(), false, true);
+ break;
+ }
+ case NYql::NDqProto::UNKNOWN:
+ [[fallthrough]];
+ case NYql::NDqProto::EPullResponseType_INT_MIN_SENTINEL_DO_NOT_USE_:
+ [[fallthrough]];
+ case NYql::NDqProto::EPullResponseType_INT_MAX_SENTINEL_DO_NOT_USE_:
+ YQL_ENSURE(false, "Unknown pull result");
+ break;
+ }
+
+ if (!Discard) {
+ auto fullResultTableEnabled = Settings->EnableFullResultWrite.Get().GetOrElse(false);
+
+ if (fullResultTableEnabled && Truncated) {
+ WriteToFullResultTable(new NDqProto::TData(std::move(*response.MutableData())));
+ } else {
+ DataParts.emplace_back(std::move(*response.MutableData()));
+
+ bool full = true;
+ bool exceedRows = false;
+ try {
+ full = ResultBuilder->WriteYsonData(DataParts.back(), [this, &exceedRows](const TString& rawYson) {
+ if (RowsLimit && Rows + 1 > *RowsLimit) {
+ exceedRows = true;
+ return false;
+ } else if (ResultYsonOut->Counter() + rawYson.size() > SizeLimit) {
+ return false;
+ }
+ ResultYsonWriter->OnListItem();
+ ResultYsonWriter->OnRaw(rawYson);
+ ++Rows;
+ return true;
+ });
+ } catch (...) {
+ OnError(CurrentExceptionMessage(), false, false);
+ return;
}
-
- if (!full) {
- if (fullResultTableEnabled) {
- FlushCurrent();
- } else {
- TString issueMsg;
- if (exceedRows) {
- issueMsg = TStringBuilder() << "Rows limit reached: " << *RowsLimit;
- } else {
- issueMsg = TStringBuilder() << "Size limit reached: " << SizeLimit;
- }
- TIssue issue(issueMsg);
- issue.Severity = TSeverityIds::S_WARNING;
- Finish(/*truncated = */ true, {issue});
- }
+
+ if (!full) {
+ if (fullResultTableEnabled) {
+ FlushCurrent();
+ } else {
+ TString issueMsg;
+ if (exceedRows) {
+ issueMsg = TStringBuilder() << "Rows limit reached: " << *RowsLimit;
+ } else {
+ issueMsg = TStringBuilder() << "Size limit reached: " << SizeLimit;
+ }
+ TIssue issue(issueMsg);
+ issue.Severity = TSeverityIds::S_WARNING;
+ Finish(/*truncated = */ true, {issue});
+ }
}
}
- }
- }
-
- void FlushCurrent() {
- YQL_LOG(DEBUG) << __FUNCTION__;
- YQL_ENSURE(!Truncated);
- YQL_ENSURE(!FullResultWriterID);
- YQL_ENSURE(Settings->EnableFullResultWrite.Get().GetOrElse(false));
-
- NDqProto::TGraphExecutionEvent record;
- record.SetEventType(NDqProto::EGraphExecutionEventType::FULL_RESULT);
- NDqProto::TGraphExecutionEvent::TFullResultDescriptor payload;
- payload.SetResultType(ResultBuilder->GetSerializedType());
- record.MutableMessage()->PackFrom(payload);
- Send(GraphExecutionEventsId, new TEvGraphExecutionEvent(record));
- Synchronize<TEvGraphExecutionEvent>([this](TEvGraphExecutionEvent::TPtr& ev) {
- Y_VERIFY(ev->Get()->Record.GetEventType() == NYql::NDqProto::EGraphExecutionEventType::SYNC);
- YQL_LOG_CTX_SCOPE(TraceId);
-
- if (auto msg = ev->Get()->Record.GetErrorMessage()) {
- OnError(msg, false, true);
- } else {
- NActorsProto::TActorId fullResultWriterProto;
- ev->Get()->Record.GetMessage().UnpackTo(&fullResultWriterProto);
- FullResultWriterID = NActors::ActorIdFromProto(fullResultWriterProto);
- Truncated = true;
- WriteAllDataPartsToFullResultTable();
+ }
+ }
+
+ void FlushCurrent() {
+ YQL_LOG(DEBUG) << __FUNCTION__;
+ YQL_ENSURE(!Truncated);
+ YQL_ENSURE(!FullResultWriterID);
+ YQL_ENSURE(Settings->EnableFullResultWrite.Get().GetOrElse(false));
+
+ NDqProto::TGraphExecutionEvent record;
+ record.SetEventType(NDqProto::EGraphExecutionEventType::FULL_RESULT);
+ NDqProto::TGraphExecutionEvent::TFullResultDescriptor payload;
+ payload.SetResultType(ResultBuilder->GetSerializedType());
+ record.MutableMessage()->PackFrom(payload);
+ Send(GraphExecutionEventsId, new TEvGraphExecutionEvent(record));
+ Synchronize<TEvGraphExecutionEvent>([this](TEvGraphExecutionEvent::TPtr& ev) {
+ Y_VERIFY(ev->Get()->Record.GetEventType() == NYql::NDqProto::EGraphExecutionEventType::SYNC);
+ YQL_LOG_CTX_SCOPE(TraceId);
+
+ if (auto msg = ev->Get()->Record.GetErrorMessage()) {
+ OnError(msg, false, true);
+ } else {
+ NActorsProto::TActorId fullResultWriterProto;
+ ev->Get()->Record.GetMessage().UnpackTo(&fullResultWriterProto);
+ FullResultWriterID = NActors::ActorIdFromProto(fullResultWriterProto);
+ Truncated = true;
+ WriteAllDataPartsToFullResultTable();
}
- });
- }
-
- bool CanSendToFullResultWriter() {
- // TODO Customize
- return FullResultSentBytes - FullResultReceivedBytes <= 32_MB;
- }
-
- template <class TCallback>
- void UpdateEventQueueStatus(TCallback callback) {
- YQL_LOG(DEBUG) << "UpdateEQStatus before: sent " << (FullResultSentBytes / 1024.0) << " kB "
- << " received " << (FullResultReceivedBytes / 1024.0) << " kB "
- << " diff " << (FullResultSentBytes - FullResultReceivedBytes) / 1024.0 << " kB";
- Send(FullResultWriterID, new TEvFullResultWriterStatusRequest());
- Synchronize<TEvFullResultWriterStatusResponse>([this, callback](TEvFullResultWriterStatusResponse::TPtr& ev) {
- YQL_LOG_CTX_SCOPE(TraceId);
- this->FullResultReceivedBytes = ev->Get()->Record.GetBytesReceived();
- YQL_LOG(DEBUG) << "UpdateEQStatus after: sent " << (FullResultSentBytes / 1024.0) << " kB "
- << " received " << (FullResultReceivedBytes / 1024.0) << " kB "
- << " diff " << (FullResultSentBytes - FullResultReceivedBytes) / 1024.0 << " kB";
- if (ev->Get()->Record.HasErrorMessage()) {
- YQL_LOG(DEBUG) << "Received error message: " << ev->Get()->Record.GetErrorMessage();
- OnError(ev->Get()->Record.GetErrorMessage(), false, false);
- return;
+ });
+ }
+
+ bool CanSendToFullResultWriter() {
+ // TODO Customize
+ return FullResultSentBytes - FullResultReceivedBytes <= 32_MB;
+ }
+
+ template <class TCallback>
+ void UpdateEventQueueStatus(TCallback callback) {
+ YQL_LOG(DEBUG) << "UpdateEQStatus before: sent " << (FullResultSentBytes / 1024.0) << " kB "
+ << " received " << (FullResultReceivedBytes / 1024.0) << " kB "
+ << " diff " << (FullResultSentBytes - FullResultReceivedBytes) / 1024.0 << " kB";
+ Send(FullResultWriterID, new TEvFullResultWriterStatusRequest());
+ Synchronize<TEvFullResultWriterStatusResponse>([this, callback](TEvFullResultWriterStatusResponse::TPtr& ev) {
+ YQL_LOG_CTX_SCOPE(TraceId);
+ this->FullResultReceivedBytes = ev->Get()->Record.GetBytesReceived();
+ YQL_LOG(DEBUG) << "UpdateEQStatus after: sent " << (FullResultSentBytes / 1024.0) << " kB "
+ << " received " << (FullResultReceivedBytes / 1024.0) << " kB "
+ << " diff " << (FullResultSentBytes - FullResultReceivedBytes) / 1024.0 << " kB";
+ if (ev->Get()->Record.HasErrorMessage()) {
+ YQL_LOG(DEBUG) << "Received error message: " << ev->Get()->Record.GetErrorMessage();
+ OnError(ev->Get()->Record.GetErrorMessage(), false, false);
+ return;
}
- callback();
- });
- }
-
- void WriteAllDataPartsToFullResultTable() {
- while (FullResultSentDataParts < DataParts.size() && CanSendToFullResultWriter()) {
- UnsafeWriteToFullResultTable(DataParts[FullResultSentDataParts]);
- DataParts[FullResultSentDataParts].Clear();
- ++FullResultSentDataParts;
- }
- if (FullResultSentDataParts == DataParts.size()) {
- return;
- }
- // here we cannot continue since the event queue is overloaded
- // kind of tail recursion (but without recursion)
- UpdateEventQueueStatus([this]() {
- WriteAllDataPartsToFullResultTable();
- });
- }
-
- void WriteToFullResultTable(TAutoPtr<NDqProto::TData> data) {
- if (CanSendToFullResultWriter()) {
- UnsafeWriteToFullResultTable(*data);
- return;
- }
- UpdateEventQueueStatus([this, data]() {
- WriteToFullResultTable(data);
- });
- }
-
- void UnsafeWriteToFullResultTable(const NDqProto::TData& data) {
- NDqProto::TPullResponse response;
- response.SetResponseType(EPullResponseType::CONTINUE);
- response.MutableData()->CopyFrom(data);
- ui64 respSize = response.ByteSizeLong();
- Send(FullResultWriterID, MakeHolder<TEvPullDataResponse>(response));
- FullResultSentBytes += respSize;
- }
-
- void Finish(bool truncated = false, const TIssues& issues = {}) {
- YQL_LOG(DEBUG) << __FUNCTION__ << ", truncated=" << truncated;
- YQL_ENSURE(!FinishCalled);
- FinishCalled = true;
- FinishTruncated = truncated;
- FinishIssues = issues;
- if (FullResultWriterID) {
- NDqProto::TPullResponse response;
- response.SetResponseType(EPullResponseType::FINISH);
- Send(FullResultWriterID, MakeHolder<TEvPullDataResponse>(response));
- } else {
- DoFinish();
- }
- }
-
- void DoFinish() {
- Send(ExecuterID, new TEvGraphFinished());
- }
-
- void OnFullResultWriterResponse(TEvDqFailure::TPtr& ev, const TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceId);
- YQL_LOG(DEBUG) << __FUNCTION__;
- if (ev->Get()->Record.IssuesSize() == 0) {
- DoFinish();
- } else {
- Send(ExecuterID, ev->Release().Release());
- }
- }
-
- void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceId);
- YQL_ENSURE(!ev->Get()->Record.HasResultSet() && ev->Get()->Record.GetYson().empty());
- YQL_LOG(DEBUG) << "Shutting down TResultAggregator";
-
- BlockingActors.clear();
- if (FullResultWriterID) {
- BlockingActors.insert(FullResultWriterID);
- Send(FullResultWriterID, MakeHolder<TEvents::TEvPoison>());
- }
-
- YQL_LOG(DEBUG) << "Waiting for " << BlockingActors.size() << " blocking actors";
-
- QueryResponse.Reset(ev->Release().Release());
- Become(&TResultAggregator::ShutdownHandler);
- Send(SelfId(), MakeHolder<TEvents::TEvGone>());
- }
-
- void OnShutdownQueryResult(TEvents::TEvGone::TPtr& ev, const TActorContext&) {
- YQL_LOG_CTX_SCOPE(TraceId);
- auto iter = BlockingActors.find(ev->Sender);
- if (iter != BlockingActors.end()) {
- BlockingActors.erase(iter);
- }
-
- YQL_LOG(DEBUG) << "Shutting down TResultAggregator, " << BlockingActors.size() << " blocking actors left";
-
- if (BlockingActors.empty()) {
- EndOnQueryResult();
- }
- }
-
- void EndOnQueryResult() {
- YQL_LOG(DEBUG) << __FUNCTION__;
- NDqProto::TQueryResponse result = QueryResponse->Record;
-
- YQL_ENSURE(!result.HasResultSet() && result.GetYson().empty());
- FlushCounters(result);
-
- if (ResultYsonWriter) {
- ResultYsonWriter->OnEndList();
- ResultYsonWriter.Destroy();
- }
- ResultYsonOut.Destroy();
-
- *result.MutableYson() = ResultYson;
-
- if (!FinishIssues.Empty()) {
- IssuesToMessage(FinishIssues, result.MutableIssues());
- }
- result.SetTruncated(FinishTruncated);
-
- Send(ExecuterID, new TEvQueryResponse(std::move(result)));
- }
-
- const NActors::TActorId ExecuterID;
- NActors::TActorId SourceID;
- const NActors::TActorId GraphExecutionEventsId;
- const bool Discard;
- TVector<NDqProto::TData> DataParts;
- const TString TraceId;
- TDqConfiguration::TPtr Settings;
- TDuration PullRequestTimeout;
- TDuration PingTimeout;
- TDuration PingPeriod;
- TInstant PingStartTime;
- TInstant PullRequestStartTime;
- bool PingRequested = false;
- NActors::TSchedulerCookieHolder TimerCookieHolder;
- ui64 SizeLimit = 64000000; // GRPC limit
- TMaybe<ui64> RowsLimit;
- ui64 Rows = 0;
- bool Truncated = false;
- NActors::TActorId FullResultWriterID;
- THolder<TProtoBuilder> ResultBuilder;
- TString ResultYson;
- THolder<TCountingOutput> ResultYsonOut;
- THolder<NYson::TYsonWriter> ResultYsonWriter;
- ui64 FullResultSentBytes = 0;
- ui64 FullResultReceivedBytes = 0;
- ui64 FullResultSentDataParts = 0;
-
- TIssues FinishIssues;
- bool FinishTruncated = false;
- bool FinishCalled = false;
-
- THashSet<TActorId> BlockingActors;
- THolder<TEvQueryResponse> QueryResponse;
-};
-
-class TResultPrinter: public TActor<TResultPrinter> {
-public:
+ callback();
+ });
+ }
+
+ void WriteAllDataPartsToFullResultTable() {
+ while (FullResultSentDataParts < DataParts.size() && CanSendToFullResultWriter()) {
+ UnsafeWriteToFullResultTable(DataParts[FullResultSentDataParts]);
+ DataParts[FullResultSentDataParts].Clear();
+ ++FullResultSentDataParts;
+ }
+ if (FullResultSentDataParts == DataParts.size()) {
+ return;
+ }
+ // here we cannot continue since the event queue is overloaded
+ // kind of tail recursion (but without recursion)
+ UpdateEventQueueStatus([this]() {
+ WriteAllDataPartsToFullResultTable();
+ });
+ }
+
+ void WriteToFullResultTable(TAutoPtr<NDqProto::TData> data) {
+ if (CanSendToFullResultWriter()) {
+ UnsafeWriteToFullResultTable(*data);
+ return;
+ }
+ UpdateEventQueueStatus([this, data]() {
+ WriteToFullResultTable(data);
+ });
+ }
+
+ void UnsafeWriteToFullResultTable(const NDqProto::TData& data) {
+ NDqProto::TPullResponse response;
+ response.SetResponseType(EPullResponseType::CONTINUE);
+ response.MutableData()->CopyFrom(data);
+ ui64 respSize = response.ByteSizeLong();
+ Send(FullResultWriterID, MakeHolder<TEvPullDataResponse>(response));
+ FullResultSentBytes += respSize;
+ }
+
+ void Finish(bool truncated = false, const TIssues& issues = {}) {
+ YQL_LOG(DEBUG) << __FUNCTION__ << ", truncated=" << truncated;
+ YQL_ENSURE(!FinishCalled);
+ FinishCalled = true;
+ FinishTruncated = truncated;
+ FinishIssues = issues;
+ if (FullResultWriterID) {
+ NDqProto::TPullResponse response;
+ response.SetResponseType(EPullResponseType::FINISH);
+ Send(FullResultWriterID, MakeHolder<TEvPullDataResponse>(response));
+ } else {
+ DoFinish();
+ }
+ }
+
+ void DoFinish() {
+ Send(ExecuterID, new TEvGraphFinished());
+ }
+
+ void OnFullResultWriterResponse(TEvDqFailure::TPtr& ev, const TActorContext&) {
+ YQL_LOG_CTX_SCOPE(TraceId);
+ YQL_LOG(DEBUG) << __FUNCTION__;
+ if (ev->Get()->Record.IssuesSize() == 0) {
+ DoFinish();
+ } else {
+ Send(ExecuterID, ev->Release().Release());
+ }
+ }
+
+ void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) {
+ YQL_LOG_CTX_SCOPE(TraceId);
+ YQL_ENSURE(!ev->Get()->Record.HasResultSet() && ev->Get()->Record.GetYson().empty());
+ YQL_LOG(DEBUG) << "Shutting down TResultAggregator";
+
+ BlockingActors.clear();
+ if (FullResultWriterID) {
+ BlockingActors.insert(FullResultWriterID);
+ Send(FullResultWriterID, MakeHolder<TEvents::TEvPoison>());
+ }
+
+ YQL_LOG(DEBUG) << "Waiting for " << BlockingActors.size() << " blocking actors";
+
+ QueryResponse.Reset(ev->Release().Release());
+ Become(&TResultAggregator::ShutdownHandler);
+ Send(SelfId(), MakeHolder<TEvents::TEvGone>());
+ }
+
+ void OnShutdownQueryResult(TEvents::TEvGone::TPtr& ev, const TActorContext&) {
+ YQL_LOG_CTX_SCOPE(TraceId);
+ auto iter = BlockingActors.find(ev->Sender);
+ if (iter != BlockingActors.end()) {
+ BlockingActors.erase(iter);
+ }
+
+ YQL_LOG(DEBUG) << "Shutting down TResultAggregator, " << BlockingActors.size() << " blocking actors left";
+
+ if (BlockingActors.empty()) {
+ EndOnQueryResult();
+ }
+ }
+
+ void EndOnQueryResult() {
+ YQL_LOG(DEBUG) << __FUNCTION__;
+ NDqProto::TQueryResponse result = QueryResponse->Record;
+
+ YQL_ENSURE(!result.HasResultSet() && result.GetYson().empty());
+ FlushCounters(result);
+
+ if (ResultYsonWriter) {
+ ResultYsonWriter->OnEndList();
+ ResultYsonWriter.Destroy();
+ }
+ ResultYsonOut.Destroy();
+
+ *result.MutableYson() = ResultYson;
+
+ if (!FinishIssues.Empty()) {
+ IssuesToMessage(FinishIssues, result.MutableIssues());
+ }
+ result.SetTruncated(FinishTruncated);
+
+ Send(ExecuterID, new TEvQueryResponse(std::move(result)));
+ }
+
+ const NActors::TActorId ExecuterID;
+ NActors::TActorId SourceID;
+ const NActors::TActorId GraphExecutionEventsId;
+ const bool Discard;
+ TVector<NDqProto::TData> DataParts;
+ const TString TraceId;
+ TDqConfiguration::TPtr Settings;
+ TDuration PullRequestTimeout;
+ TDuration PingTimeout;
+ TDuration PingPeriod;
+ TInstant PingStartTime;
+ TInstant PullRequestStartTime;
+ bool PingRequested = false;
+ NActors::TSchedulerCookieHolder TimerCookieHolder;
+ ui64 SizeLimit = 64000000; // GRPC limit
+ TMaybe<ui64> RowsLimit;
+ ui64 Rows = 0;
+ bool Truncated = false;
+ NActors::TActorId FullResultWriterID;
+ THolder<TProtoBuilder> ResultBuilder;
+ TString ResultYson;
+ THolder<TCountingOutput> ResultYsonOut;
+ THolder<NYson::TYsonWriter> ResultYsonWriter;
+ ui64 FullResultSentBytes = 0;
+ ui64 FullResultReceivedBytes = 0;
+ ui64 FullResultSentDataParts = 0;
+
+ TIssues FinishIssues;
+ bool FinishTruncated = false;
+ bool FinishCalled = false;
+
+ THashSet<TActorId> BlockingActors;
+ THolder<TEvQueryResponse> QueryResponse;
+};
+
+class TResultPrinter: public TActor<TResultPrinter> {
+public:
static constexpr char ActorName[] = "YQL_DQ_RESULT_PRINTER";
- TResultPrinter(IOutputStream& output, NThreading::TPromise<void>& promise)
- : TActor<TResultPrinter>(&TResultPrinter::Handler)
- , Output(output)
- , Promise(promise)
- {
- }
-
-private:
- STRICT_STFUNC(Handler, { HFunc(TEvQueryResponse, OnQueryResult); })
-
- void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) {
- if (!ev->Get()->Record.HasResultSet()&&ev->Get()->Record.GetYson().empty()) {
- NYql::TIssues issues;
- NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues);
- Cerr << issues.ToString() << Endl;
- } else {
- auto ysonString = !ev->Get()->Record.GetYson().empty()
- ? ev->Get()->Record.GetYson()
- : NYdb::FormatResultSetYson(ev->Get()->Record.GetResultSet(), NYson::EYsonFormat::Binary);
- auto ysonNode = NYT::NodeFromYsonString(ysonString, NYson::EYsonType::Node);
- YQL_ENSURE(ysonNode.GetType() == NYT::TNode::EType::List);
- for (const auto& row : ysonNode.AsList()) {
- Output << NYT::NodeToYsonString(row) << "\n";
+ TResultPrinter(IOutputStream& output, NThreading::TPromise<void>& promise)
+ : TActor<TResultPrinter>(&TResultPrinter::Handler)
+ , Output(output)
+ , Promise(promise)
+ {
+ }
+
+private:
+ STRICT_STFUNC(Handler, { HFunc(TEvQueryResponse, OnQueryResult); })
+
+ void OnQueryResult(TEvQueryResponse::TPtr& ev, const TActorContext&) {
+ if (!ev->Get()->Record.HasResultSet()&&ev->Get()->Record.GetYson().empty()) {
+ NYql::TIssues issues;
+ NYql::IssuesFromMessage(ev->Get()->Record.GetIssues(), issues);
+ Cerr << issues.ToString() << Endl;
+ } else {
+ auto ysonString = !ev->Get()->Record.GetYson().empty()
+ ? ev->Get()->Record.GetYson()
+ : NYdb::FormatResultSetYson(ev->Get()->Record.GetResultSet(), NYson::EYsonFormat::Binary);
+ auto ysonNode = NYT::NodeFromYsonString(ysonString, NYson::EYsonType::Node);
+ YQL_ENSURE(ysonNode.GetType() == NYT::TNode::EType::List);
+ for (const auto& row : ysonNode.AsList()) {
+ Output << NYT::NodeToYsonString(row) << "\n";
}
- }
-
- Promise.SetValue();
- PassAway();
- }
-
-private:
- IOutputStream& Output;
- NThreading::TPromise<void>& Promise;
-};
-
-} // unnamed
-
-THolder<NActors::IActor> MakeResultAggregator(
- const TVector<TString>& columns,
- const NActors::TActorId& executerId,
- const TString& traceId,
- const THashMap<TString, TString>& secureParams,
- const TDqConfiguration::TPtr& settings,
- const TString& resultType,
- bool discard,
- const NActors::TActorId& graphExecutionEventsId)
-{
- THolder<IActor> result;
- if (!settings->EnableComputeActor.Get().GetOrElse(false)) {
- // worker actor pull
- result = MakeHolder<TResultAggregator>(columns, executerId, traceId, settings, resultType, graphExecutionEventsId, discard);
- } else {
- // compute actor push
- result = NYql::MakeResultReceiver(columns, executerId, traceId, settings, secureParams, resultType, discard);
+ }
+
+ Promise.SetValue();
+ PassAway();
+ }
+
+private:
+ IOutputStream& Output;
+ NThreading::TPromise<void>& Promise;
+};
+
+} // unnamed
+
+THolder<NActors::IActor> MakeResultAggregator(
+ const TVector<TString>& columns,
+ const NActors::TActorId& executerId,
+ const TString& traceId,
+ const THashMap<TString, TString>& secureParams,
+ const TDqConfiguration::TPtr& settings,
+ const TString& resultType,
+ bool discard,
+ const NActors::TActorId& graphExecutionEventsId)
+{
+ THolder<IActor> result;
+ if (!settings->EnableComputeActor.Get().GetOrElse(false)) {
+ // worker actor pull
+ result = MakeHolder<TResultAggregator>(columns, executerId, traceId, settings, resultType, graphExecutionEventsId, discard);
+ } else {
+ // compute actor push
+ result = NYql::MakeResultReceiver(columns, executerId, traceId, settings, secureParams, resultType, discard);
}
- return MakeHolder<TLogWrapReceive>(result.Release(), traceId);
-}
+ return MakeHolder<TLogWrapReceive>(result.Release(), traceId);
+}
-} // NYql::NDqs::NExecutionHelpers
+} // NYql::NDqs::NExecutionHelpers
diff --git a/ydb/library/yql/providers/dq/actors/result_aggregator.h b/ydb/library/yql/providers/dq/actors/result_aggregator.h
index 450bc2de88..2c22fbf73c 100644
--- a/ydb/library/yql/providers/dq/actors/result_aggregator.h
+++ b/ydb/library/yql/providers/dq/actors/result_aggregator.h
@@ -15,6 +15,6 @@ namespace NYql::NDqs::NExecutionHelpers {
const THashMap<TString, TString>& secureParams,
const TDqConfiguration::TPtr& settings,
const TString& resultType,
- bool discard,
- const NActors::TActorId& graphExecutionEventsId);
+ bool discard,
+ const NActors::TActorId& graphExecutionEventsId);
} // namespace NYql::NDqs::NExecutionHelpers
diff --git a/ydb/library/yql/providers/dq/actors/result_receiver.cpp b/ydb/library/yql/providers/dq/actors/result_receiver.cpp
index 7fdb73f257..be4b8e28d2 100644
--- a/ydb/library/yql/providers/dq/actors/result_receiver.cpp
+++ b/ydb/library/yql/providers/dq/actors/result_receiver.cpp
@@ -1,5 +1,5 @@
#include "result_receiver.h"
-#include "proto_builder.h"
+#include "proto_builder.h"
#include <ydb/library/yql/providers/dq/actors/execution_helpers.h>
#include <ydb/library/yql/providers/dq/actors/events.h>
diff --git a/ydb/library/yql/providers/dq/actors/task_controller.cpp b/ydb/library/yql/providers/dq/actors/task_controller.cpp
index 238828e453..28b80268bb 100644
--- a/ydb/library/yql/providers/dq/actors/task_controller.cpp
+++ b/ydb/library/yql/providers/dq/actors/task_controller.cpp
@@ -1,10 +1,10 @@
#include "task_controller.h"
#include "execution_helpers.h"
#include "events.h"
-#include "proto_builder.h"
+#include "proto_builder.h"
#include "actor_helpers.h"
#include "executer_actor.h"
-
+
#include <ydb/library/yql/providers/dq/counters/counters.h>
#include <ydb/library/yql/providers/dq/common/yql_dq_common.h>
diff --git a/ydb/library/yql/providers/dq/actors/worker_actor.cpp b/ydb/library/yql/providers/dq/actors/worker_actor.cpp
index f4503a60b2..2a3712f4ea 100644
--- a/ydb/library/yql/providers/dq/actors/worker_actor.cpp
+++ b/ydb/library/yql/providers/dq/actors/worker_actor.cpp
@@ -2,7 +2,7 @@
#include <ydb/library/yql/providers/dq/task_runner_actor/task_runner_actor.h>
#include <ydb/library/yql/providers/dq/runtime/runtime_data.h>
-
+
#include <ydb/library/yql/utils/failure_injector/failure_injector.h>
#include <ydb/library/yql/utils/actor_log/log.h>
#include <ydb/library/yql/utils/log/log.h>
@@ -285,7 +285,7 @@ private:
YQL_LOG_CTX_SCOPE(TraceId);
YQL_LOG(DEBUG) << "TDqWorker::OnDqTask";
- TFailureInjector::Reach("dq_task_failure", [] {::_exit(1); });
+ TFailureInjector::Reach("dq_task_failure", [] {::_exit(1); });
Y_VERIFY(!TaskRunnerActor);
diff --git a/ydb/library/yql/providers/dq/actors/ya.make b/ydb/library/yql/providers/dq/actors/ya.make
index 14e93b587d..4132f28320 100644
--- a/ydb/library/yql/providers/dq/actors/ya.make
+++ b/ydb/library/yql/providers/dq/actors/ya.make
@@ -11,15 +11,15 @@ SRCS(
resource_allocator.cpp
task_controller.cpp
worker_actor.cpp
- result_aggregator.cpp
- result_receiver.cpp
- full_result_writer.cpp
- proto_builder.cpp
+ result_aggregator.cpp
+ result_receiver.cpp
+ full_result_writer.cpp
+ proto_builder.cpp
)
PEERDIR(
library/cpp/actors/core
- library/cpp/yson
+ library/cpp/yson
ydb/core/base
ydb/library/mkql_proto
ydb/library/yql/core/services
@@ -48,6 +48,6 @@ PEERDIR(
ydb/library/yql/providers/dq/worker_manager/interface
)
-YQL_LAST_ABI_VERSION()
+YQL_LAST_ABI_VERSION()
END()
diff --git a/ydb/library/yql/providers/dq/api/protos/dqs.proto b/ydb/library/yql/providers/dq/api/protos/dqs.proto
index 09aecfcf92..804f38cc2c 100644
--- a/ydb/library/yql/providers/dq/api/protos/dqs.proto
+++ b/ydb/library/yql/providers/dq/api/protos/dqs.proto
@@ -210,7 +210,7 @@ enum EGraphExecutionEventType {
START = 1;
FAIL = 2;
SUCCESS = 3;
- FULL_RESULT = 4;
+ FULL_RESULT = 4;
}
message TGraphExecutionEvent {
@@ -223,13 +223,13 @@ message TGraphExecutionEvent {
}
message TExecuteGraphDescriptor {
- TMap SecureParams = 1;
- TMap GraphParams = 2;
- }
-
- message TFullResultDescriptor {
- bytes ResultType = 1;
+ TMap SecureParams = 1;
+ TMap GraphParams = 2;
}
+
+ message TFullResultDescriptor {
+ bytes ResultType = 1;
+ }
}
message TFullResultWriterStatusRequest {
diff --git a/ydb/library/yql/providers/dq/api/protos/service.proto b/ydb/library/yql/providers/dq/api/protos/service.proto
index b91351e5d1..63319401c7 100644
--- a/ydb/library/yql/providers/dq/api/protos/service.proto
+++ b/ydb/library/yql/providers/dq/api/protos/service.proto
@@ -80,7 +80,7 @@ message ExecuteGraphRequest {
repeated TAttr Settings = 8;
map<string, string> SecureParams = 9;
bool Discard = 12;
- map<string, string> GraphParams = 13;
+ map<string, string> GraphParams = 13;
}
message ExecuteGraphResponse {
@@ -310,8 +310,8 @@ message GetMasterResponse {
message ConfigureFailureInjectorRequest {
string Name = 1;
- uint64 Skip = 2;
- uint64 CountOfFails = 5;
+ uint64 Skip = 2;
+ uint64 CountOfFails = 5;
uint32 NodeId = 4; // aka actor id
}
diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp b/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp
index f9ba89a33d..0db57b3f33 100644
--- a/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp
+++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.cpp
@@ -19,12 +19,12 @@ TDqConfiguration::TDqConfiguration() {
REGISTER_SETTING(*this, FallbackPolicy);
REGISTER_SETTING(*this, PullRequestTimeoutMs);
REGISTER_SETTING(*this, PingTimeoutMs);
- REGISTER_SETTING(*this, UseSimpleYtReader);
+ REGISTER_SETTING(*this, UseSimpleYtReader);
REGISTER_SETTING(*this, OptLLVM);
REGISTER_SETTING(*this, ChannelBufferSize);
REGISTER_SETTING(*this, OutputChunkMaxSize);
- REGISTER_SETTING(*this, MemoryLimit);
- REGISTER_SETTING(*this, EnableInsert);
+ REGISTER_SETTING(*this, MemoryLimit);
+ REGISTER_SETTING(*this, EnableInsert);
REGISTER_SETTING(*this, _LiteralTimeout);
REGISTER_SETTING(*this, _TableTimeout);
diff --git a/ydb/library/yql/providers/dq/common/yql_dq_settings.h b/ydb/library/yql/providers/dq/common/yql_dq_settings.h
index 3c37e4d48f..bdda899eeb 100644
--- a/ydb/library/yql/providers/dq/common/yql_dq_settings.h
+++ b/ydb/library/yql/providers/dq/common/yql_dq_settings.h
@@ -5,8 +5,8 @@
#include <ydb/library/yql/core/yql_data_provider.h>
-#include <library/cpp/string_utils/parse_size/parse_size.h>
-
+#include <library/cpp/string_utils/parse_size/parse_size.h>
+
#include <util/generic/size_literals.h>
#include <util/random/random.h>
@@ -19,7 +19,7 @@ struct TDqSettings {
static constexpr ui32 MaxTasksPerOperation = 70U;
static constexpr ui64 PortoMemoryLimit = 3_GB;
static constexpr bool EnablePorto = false;
- static constexpr ui64 DataSizePerJob = 128_MB;
+ static constexpr ui64 DataSizePerJob = 128_MB;
static constexpr ui64 MaxDataSizePerJob = 600_MB;
static constexpr int MaxNetworkRetries = 5;
static constexpr ui64 LiteralTimeout = 60000; // 1 minutes
@@ -45,16 +45,16 @@ struct TDqSettings {
NCommon::TConfSetting<TString, false> FallbackPolicy;
NCommon::TConfSetting<ui64, false> PullRequestTimeoutMs;
NCommon::TConfSetting<ui64, false> PingTimeoutMs;
- NCommon::TConfSetting<bool, false> UseSimpleYtReader;
+ NCommon::TConfSetting<bool, false> UseSimpleYtReader;
NCommon::TConfSetting<TString, false> OptLLVM;
NCommon::TConfSetting<ui64, false> ChannelBufferSize;
NCommon::TConfSetting<ui64, false> OutputChunkMaxSize;
- NCommon::TConfSetting<NSize::TSize, false> MemoryLimit;
+ NCommon::TConfSetting<NSize::TSize, false> MemoryLimit;
NCommon::TConfSetting<ui64, false> _LiteralTimeout;
NCommon::TConfSetting<ui64, false> _TableTimeout;
NCommon::TConfSetting<ui64, false> _LongWorkersAllocationWarnTimeout;
NCommon::TConfSetting<ui64, false> _LongWorkersAllocationFailTimeout;
- NCommon::TConfSetting<bool, false> EnableInsert;
+ NCommon::TConfSetting<bool, false> EnableInsert;
NCommon::TConfSetting<ui64, false> _AllResultsBytesLimit;
NCommon::TConfSetting<ui64, false> _RowsLimitPerWrite;
NCommon::TConfSetting<bool, false> EnableStrip;
@@ -90,7 +90,7 @@ struct TDqSettings {
SAVE_SETTING(OptLLVM);
SAVE_SETTING(ChannelBufferSize);
SAVE_SETTING(OutputChunkMaxSize);
- SAVE_SETTING(MemoryLimit);
+ SAVE_SETTING(MemoryLimit);
SAVE_SETTING(_LiteralTimeout);
SAVE_SETTING(_TableTimeout);
SAVE_SETTING(_LongWorkersAllocationWarnTimeout);
@@ -131,9 +131,9 @@ struct TDqConfiguration: public TDqSettings, public NCommon::TSettingDispatcher
void Init(const TProtoConfig& config, const TString& userName)
{
// Init settings from config
- this->Dispatch(config.GetDefaultSettings(), userName);
+ this->Dispatch(config.GetDefaultSettings(), userName);
- // TODO: drop after releasing new gateways config
+ // TODO: drop after releasing new gateways config
if (this->AnalyzeQuery.Get().Empty()) {
int percent = 0;
diff --git a/ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.json b/ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.json
index 566f8d0503..5c778f9146 100644
--- a/ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.json
+++ b/ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.json
@@ -6,22 +6,22 @@
"FreeArgBuilderBase": "TFreeArgCallableBuilderBase",
"Nodes": [
{
- "Name": "TDqReadWrapBase",
- "Base": "TExprBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
- "Children": [
- {"Index": 0, "Name": "Input", "Type": "TExprBase"},
+ "Name": "TDqReadWrapBase",
+ "Base": "TExprBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
+ "Children": [
+ {"Index": 0, "Name": "Input", "Type": "TExprBase"},
{"Index": 1, "Name": "Token", "Type": "TCoSecureParam", "Optional": true},
{"Index": 2, "Name": "Flags", "Type": "TCoAtomList", "Optional": true}
- ]
- },
- {
- "Name": "TDqReadWrap",
- "Base": "TDqReadWrapBase",
- "Match": {"Type": "Callable", "Name": "DqReadWrap"}
- },
- {
+ ]
+ },
+ {
+ "Name": "TDqReadWrap",
+ "Base": "TDqReadWrapBase",
+ "Match": {"Type": "Callable", "Name": "DqReadWrap"}
+ },
+ {
"Name": "TDqReadWideWrap",
"Base": "TDqReadWrapBase",
"Match": {"Type": "Callable", "Name": "DqReadWideWrap"}
diff --git a/ydb/library/yql/providers/dq/interface/ya.make b/ydb/library/yql/providers/dq/interface/ya.make
index 8d72c16c03..870bb582ab 100644
--- a/ydb/library/yql/providers/dq/interface/ya.make
+++ b/ydb/library/yql/providers/dq/interface/ya.make
@@ -1,20 +1,20 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:yql)
-
-SRCS(
- yql_dq_integration.cpp
+
+SRCS(
+ yql_dq_integration.cpp
yql_dq_task_preprocessor.cpp
- yql_dq_task_transform.cpp
-)
-
-PEERDIR(
+ yql_dq_task_transform.cpp
+)
+
+PEERDIR(
contrib/libs/protobuf
- library/cpp/yson
+ library/cpp/yson
ydb/library/yql/ast
ydb/library/yql/core
-)
-
-YQL_LAST_ABI_VERSION()
+)
+
+YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.cpp b/ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.cpp
index 8acb89a241..12c94eceb0 100644
--- a/ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.cpp
+++ b/ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.cpp
@@ -1 +1 @@
-#include "yql_dq_full_result_writer.h"
+#include "yql_dq_full_result_writer.h"
diff --git a/ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.h b/ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.h
index 38115ba822..4372669dea 100644
--- a/ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.h
+++ b/ydb/library/yql/providers/dq/interface/yql_dq_full_result_writer.h
@@ -1,16 +1,16 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/public/udf/udf_value.h>
-
-namespace NYql {
-
-class IDqFullResultWriter {
-public:
- virtual ~IDqFullResultWriter() = default;
- virtual void AddRow(const NYql::NUdf::TUnboxedValuePod& row) = 0;
- virtual void Finish() = 0;
- virtual void Abort() = 0;
- virtual ui64 GetRowCount() const = 0;
-};
-
-} // NYql
+
+namespace NYql {
+
+class IDqFullResultWriter {
+public:
+ virtual ~IDqFullResultWriter() = default;
+ virtual void AddRow(const NYql::NUdf::TUnboxedValuePod& row) = 0;
+ virtual void Finish() = 0;
+ virtual void Abort() = 0;
+ virtual ui64 GetRowCount() const = 0;
+};
+
+} // NYql
diff --git a/ydb/library/yql/providers/dq/interface/yql_dq_integration.cpp b/ydb/library/yql/providers/dq/interface/yql_dq_integration.cpp
index acd0ae0ecb..434328b145 100644
--- a/ydb/library/yql/providers/dq/interface/yql_dq_integration.cpp
+++ b/ydb/library/yql/providers/dq/interface/yql_dq_integration.cpp
@@ -1 +1 @@
-#include "yql_dq_integration.h"
+#include "yql_dq_integration.h"
diff --git a/ydb/library/yql/providers/dq/interface/yql_dq_integration.h b/ydb/library/yql/providers/dq/interface/yql_dq_integration.h
index ee1938e698..0a85530407 100644
--- a/ydb/library/yql/providers/dq/interface/yql_dq_integration.h
+++ b/ydb/library/yql/providers/dq/interface/yql_dq_integration.h
@@ -1,43 +1,43 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/core/yql_data_provider.h>
#include <ydb/library/yql/ast/yql_expr.h>
-
-#include <library/cpp/yson/writer.h>
-
-#include <util/generic/string.h>
-#include <util/generic/vector.h>
-#include <util/generic/maybe.h>
-
+
+#include <library/cpp/yson/writer.h>
+
+#include <util/generic/string.h>
+#include <util/generic/vector.h>
+#include <util/generic/maybe.h>
+
#include <google/protobuf/any.pb.h>
-namespace NYql {
-
+namespace NYql {
+
struct TDqSettings;
-
-namespace NCommon {
- class TMkqlCallableCompilerBase;
-}
-
-class TFallbackError: public yexception {
-};
-
-class IDqIntegration {
-public:
- virtual ~IDqIntegration() {}
-
- virtual ui64 Partition(const TDqSettings& config, size_t maxPartitions, const TExprNode& node,
- TVector<TString>& partitions, TString* clusterName, TExprContext& ctx, bool canFallback) = 0;
+
+namespace NCommon {
+ class TMkqlCallableCompilerBase;
+}
+
+class TFallbackError: public yexception {
+};
+
+class IDqIntegration {
+public:
+ virtual ~IDqIntegration() {}
+
+ virtual ui64 Partition(const TDqSettings& config, size_t maxPartitions, const TExprNode& node,
+ TVector<TString>& partitions, TString* clusterName, TExprContext& ctx, bool canFallback) = 0;
virtual TMaybe<ui64> CanRead(const TDqSettings& config, const TExprNode& read, TExprContext& ctx, bool skipIssues = true) = 0;
virtual TExprNode::TPtr WrapRead(const TDqSettings& config, const TExprNode::TPtr& read, TExprContext& ctx) = 0;
- virtual TMaybe<bool> CanWrite(const TDqSettings& config, const TExprNode& write, TExprContext& ctx) = 0;
- virtual void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) = 0;
+ virtual TMaybe<bool> CanWrite(const TDqSettings& config, const TExprNode& write, TExprContext& ctx) = 0;
+ virtual void RegisterMkqlCompiler(NCommon::TMkqlCallableCompilerBase& compiler) = 0;
virtual bool CanFallback() = 0;
virtual void FillSourceSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sourceType) = 0;
virtual void FillSinkSettings(const TExprNode& node, ::google::protobuf::Any& settings, TString& sinkType) = 0;
- virtual void Annotate(const TExprNode& node, THashMap<TString, TString>& params) = 0;
- virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) = 0;
- virtual void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) = 0;
-};
-
-} // namespace NYql
+ virtual void Annotate(const TExprNode& node, THashMap<TString, TString>& params) = 0;
+ virtual bool PrepareFullResultTableParams(const TExprNode& root, TExprContext& ctx, THashMap<TString, TString>& params, THashMap<TString, TString>& secureParams) = 0;
+ virtual void WriteFullResultTableRef(NYson::TYsonWriter& writer, const TVector<TString>& columns, const THashMap<TString, TString>& graphParams) = 0;
+};
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.cpp b/ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.cpp
index a6d48e7a3f..a4d1db1318 100644
--- a/ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.cpp
+++ b/ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.cpp
@@ -1 +1 @@
-#include "yql_dq_task_preprocessor.h"
+#include "yql_dq_task_preprocessor.h"
diff --git a/ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.h b/ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.h
index f70d4f6a91..4e43107735 100644
--- a/ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.h
+++ b/ydb/library/yql/providers/dq/interface/yql_dq_task_preprocessor.h
@@ -1,11 +1,11 @@
#pragma once
-#include "yql_dq_full_result_writer.h"
-
+#include "yql_dq_full_result_writer.h"
+
#include <util/generic/ptr.h>
-#include <vector>
-
+#include <vector>
+
namespace NYql {
class IDqTaskPreprocessor : public TThrRefBase {
@@ -14,12 +14,12 @@ public:
virtual ~IDqTaskPreprocessor() = default;
- virtual THashMap<TString, TString> GetTaskParams(const THashMap<TString, TString>& graphParams, const THashMap<TString, TString>& secureParams) = 0;
- virtual void Finish(bool success) = 0;
- virtual THolder<IDqFullResultWriter> CreateFullResultWriter() = 0;
+ virtual THashMap<TString, TString> GetTaskParams(const THashMap<TString, TString>& graphParams, const THashMap<TString, TString>& secureParams) = 0;
+ virtual void Finish(bool success) = 0;
+ virtual THolder<IDqFullResultWriter> CreateFullResultWriter() = 0;
};
using TDqTaskPreprocessorFactory = std::function<IDqTaskPreprocessor::TPtr()>;
-using TDqTaskPreprocessorFactoryCollection = std::vector<TDqTaskPreprocessorFactory>;
+using TDqTaskPreprocessorFactoryCollection = std::vector<TDqTaskPreprocessorFactory>;
-} // namespace NYql
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/interface/yql_dq_task_transform.cpp b/ydb/library/yql/providers/dq/interface/yql_dq_task_transform.cpp
index b45ac30a73..66f76e62e9 100644
--- a/ydb/library/yql/providers/dq/interface/yql_dq_task_transform.cpp
+++ b/ydb/library/yql/providers/dq/interface/yql_dq_task_transform.cpp
@@ -1,22 +1,22 @@
#include <ydb/library/yql/providers/dq/interface/yql_dq_task_transform.h>
-
-namespace NYql {
-
-TTaskTransformFactory CreateCompositeTaskTransformFactory(TVector<TTaskTransformFactory> factories) {
- return [factories = std::move(factories)] (const THashMap<TString, TString>& taskParams, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry) -> NKikimr::NMiniKQL::TCallableVisitFuncProvider {
- TVector<NKikimr::NMiniKQL::TCallableVisitFuncProvider> funcProviders;
- for (auto& factory: factories) {
- funcProviders.push_back(factory(taskParams, funcRegistry));
- }
- return [funcProviders = std::move(funcProviders)] (const NKikimr::NMiniKQL::TInternName& name) -> NKikimr::NMiniKQL::TCallableVisitFunc {
- for (auto& provider: funcProviders) {
- if (auto res = provider(name)) {
- return res;
- }
- }
- return {};
- };
- };
-}
-
-} // NYql
+
+namespace NYql {
+
+TTaskTransformFactory CreateCompositeTaskTransformFactory(TVector<TTaskTransformFactory> factories) {
+ return [factories = std::move(factories)] (const THashMap<TString, TString>& taskParams, const NKikimr::NMiniKQL::IFunctionRegistry* funcRegistry) -> NKikimr::NMiniKQL::TCallableVisitFuncProvider {
+ TVector<NKikimr::NMiniKQL::TCallableVisitFuncProvider> funcProviders;
+ for (auto& factory: factories) {
+ funcProviders.push_back(factory(taskParams, funcRegistry));
+ }
+ return [funcProviders = std::move(funcProviders)] (const NKikimr::NMiniKQL::TInternName& name) -> NKikimr::NMiniKQL::TCallableVisitFunc {
+ for (auto& provider: funcProviders) {
+ if (auto res = provider(name)) {
+ return res;
+ }
+ }
+ return {};
+ };
+ };
+}
+
+} // NYql
diff --git a/ydb/library/yql/providers/dq/interface/yql_dq_task_transform.h b/ydb/library/yql/providers/dq/interface/yql_dq_task_transform.h
index 3077910b74..139f49c874 100644
--- a/ydb/library/yql/providers/dq/interface/yql_dq_task_transform.h
+++ b/ydb/library/yql/providers/dq/interface/yql_dq_task_transform.h
@@ -1,17 +1,17 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/minikql/mkql_node_visitor.h>
#include <ydb/library/yql/minikql/mkql_function_registry.h>
-
-#include <util/generic/hash.h>
-#include <util/generic/string.h>
-
-#include <functional>
-
-namespace NYql {
-
-using TTaskTransformFactory = std::function<NKikimr::NMiniKQL::TCallableVisitFuncProvider(const THashMap<TString, TString>&, const NKikimr::NMiniKQL::IFunctionRegistry*)>;
-
-TTaskTransformFactory CreateCompositeTaskTransformFactory(TVector<TTaskTransformFactory> factories);
-
-} // namespace NYql
+
+#include <util/generic/hash.h>
+#include <util/generic/string.h>
+
+#include <functional>
+
+namespace NYql {
+
+using TTaskTransformFactory = std::function<NKikimr::NMiniKQL::TCallableVisitFuncProvider(const THashMap<TString, TString>&, const NKikimr::NMiniKQL::IFunctionRegistry*)>;
+
+TTaskTransformFactory CreateCompositeTaskTransformFactory(TVector<TTaskTransformFactory> factories);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/local_gateway/ya.make b/ydb/library/yql/providers/dq/local_gateway/ya.make
index aa1ed08342..29bb425042 100644
--- a/ydb/library/yql/providers/dq/local_gateway/ya.make
+++ b/ydb/library/yql/providers/dq/local_gateway/ya.make
@@ -2,8 +2,8 @@ LIBRARY()
OWNER(g:yql)
-YQL_LAST_ABI_VERSION()
-
+YQL_LAST_ABI_VERSION()
+
SRCS(
yql_dq_gateway_local.cpp
)
diff --git a/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp b/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp
index 1a8d3172e4..2c918ef462 100644
--- a/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp
+++ b/ydb/library/yql/providers/dq/local_gateway/yql_dq_gateway_local.cpp
@@ -45,7 +45,7 @@ namespace {
class TLocalServiceHolder {
public:
TLocalServiceHolder(NYdb::TDriver driver, IHTTPGateway::TPtr httpGateway, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
- TTaskTransformFactory taskTransformFactory, const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories, NBus::TBindResult interconnectPort, NBus::TBindResult grpcPort)
+ TTaskTransformFactory taskTransformFactory, const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories, NBus::TBindResult interconnectPort, NBus::TBindResult grpcPort)
{
ui32 nodeId = 1;
@@ -112,17 +112,17 @@ public:
}
void CloseSession(const TString& sessionId) override {
- return Gateway->CloseSession(sessionId);
+ return Gateway->CloseSession(sessionId);
}
NThreading::TFuture<TResult>
- ExecutePlan(const TString& sessionId, NDqs::IDqsExecutionPlanner& plan, const TVector<TString>& columns,
- const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& graphParams,
- const TDqSettings::TPtr& settings,
- const TDqProgressWriter& progressWriter, const THashMap<TString, TString>& modulesMapping,
- bool discard) override
+ ExecutePlan(const TString& sessionId, NDqs::IDqsExecutionPlanner& plan, const TVector<TString>& columns,
+ const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& graphParams,
+ const TDqSettings::TPtr& settings,
+ const TDqProgressWriter& progressWriter, const THashMap<TString, TString>& modulesMapping,
+ bool discard) override
{
- return Gateway->ExecutePlan(sessionId, plan, columns, secureParams, graphParams, settings, progressWriter, modulesMapping, discard);
+ return Gateway->ExecutePlan(sessionId, plan, columns, secureParams, graphParams, settings, progressWriter, modulesMapping, discard);
}
private:
@@ -131,26 +131,26 @@ private:
};
THolder<TLocalServiceHolder> CreateLocalServiceHolder(NYdb::TDriver driver, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
+ NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
TTaskTransformFactory taskTransformFactory, const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories, IHTTPGateway::TPtr gateway,
- NBus::TBindResult interconnectPort, NBus::TBindResult grpcPort)
-{
- return MakeHolder<TLocalServiceHolder>(driver, std::move(gateway), functionRegistry, compFactory, taskTransformFactory, dqTaskPreprocessorFactories, interconnectPort, grpcPort);
+ NBus::TBindResult interconnectPort, NBus::TBindResult grpcPort)
+{
+ return MakeHolder<TLocalServiceHolder>(driver, std::move(gateway), functionRegistry, compFactory, taskTransformFactory, dqTaskPreprocessorFactories, interconnectPort, grpcPort);
}
TIntrusivePtr<IDqGateway> CreateLocalDqGateway(NYdb::TDriver driver, const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
- TTaskTransformFactory taskTransformFactory, const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories,
+ NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
+ TTaskTransformFactory taskTransformFactory, const TDqTaskPreprocessorFactoryCollection& dqTaskPreprocessorFactories,
IHTTPGateway::TPtr gateway)
-{
+{
int startPort = 31337;
TRangeWalker<int> portWalker(startPort, startPort+100);
auto interconnectPort = BindInRange(portWalker)[1];
auto grpcPort = BindInRange(portWalker)[1];
return new TDqGatewayLocal(
- CreateLocalServiceHolder(driver, functionRegistry, compFactory, taskTransformFactory, dqTaskPreprocessorFactories, std::move(gateway), interconnectPort, grpcPort),
- CreateDqGateway(std::get<0>(NDqs::GetLocalAddress()), grpcPort.Addr.GetPort(), 8));
+ CreateLocalServiceHolder(driver, functionRegistry, compFactory, taskTransformFactory, dqTaskPreprocessorFactories, std::move(gateway), interconnectPort, grpcPort),
+ CreateDqGateway(std::get<0>(NDqs::GetLocalAddress()), grpcPort.Addr.GetPort(), 8));
}
} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/mkql/dqs_mkql_compiler.cpp b/ydb/library/yql/providers/dq/mkql/dqs_mkql_compiler.cpp
index 77820d1d5e..e40859c125 100644
--- a/ydb/library/yql/providers/dq/mkql/dqs_mkql_compiler.cpp
+++ b/ydb/library/yql/providers/dq/mkql/dqs_mkql_compiler.cpp
@@ -1,27 +1,27 @@
-#include "dqs_mkql_compiler.h"
-
+#include "dqs_mkql_compiler.h"
+
#include <ydb/library/yql/providers/dq/interface/yql_dq_integration.h>
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
#include <ydb/library/yql/providers/common/mkql/yql_provider_mkql.h>
-
-namespace NYql::NDqs {
-
-using namespace NKikimr::NMiniKQL;
-using namespace NNodes;
-
-void RegisterDqsMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler, const TTypeAnnotationContext& ctx) {
+
+namespace NYql::NDqs {
+
+using namespace NKikimr::NMiniKQL;
+using namespace NNodes;
+
+void RegisterDqsMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler, const TTypeAnnotationContext& ctx) {
compiler.AddCallable({TDqSourceWideWrap::CallableName(), TDqReadWideWrap::CallableName()},
[](const TExprNode& node, NCommon::TMkqlBuildContext&) {
- YQL_ENSURE(false, "Unsupported reader: " << node.Head().Content());
- return TRuntimeNode();
- });
-
+ YQL_ENSURE(false, "Unsupported reader: " << node.Head().Content());
+ return TRuntimeNode();
+ });
+
std::unordered_set<IDqIntegration*> integrations(ctx.DataSources.size() + ctx.DataSinks.size());
for (const auto& ds: ctx.DataSources) {
if (const auto dq = ds->GetDqIntegration()) {
integrations.emplace(dq);
- }
- }
+ }
+ }
for (const auto& ds: ctx.DataSinks) {
if (const auto dq = ds->GetDqIntegration()) {
integrations.emplace(dq);
@@ -30,4 +30,4 @@ void RegisterDqsMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler, cons
std::for_each(integrations.cbegin(), integrations.cend(), std::bind(&IDqIntegration::RegisterMkqlCompiler, std::placeholders::_1, std::ref(compiler)));
}
-}
+}
diff --git a/ydb/library/yql/providers/dq/mkql/dqs_mkql_compiler.h b/ydb/library/yql/providers/dq/mkql/dqs_mkql_compiler.h
index 360d5003ce..6f2735e83f 100644
--- a/ydb/library/yql/providers/dq/mkql/dqs_mkql_compiler.h
+++ b/ydb/library/yql/providers/dq/mkql/dqs_mkql_compiler.h
@@ -1,14 +1,14 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/providers/common/mkql/yql_provider_mkql.h>
#include <ydb/library/yql/core/yql_type_annotation.h>
-
-#include <util/generic/maybe.h>
-
-#include <functional>
-
-namespace NYql::NDqs {
-
-void RegisterDqsMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler, const TTypeAnnotationContext& ctx);
-
-}
+
+#include <util/generic/maybe.h>
+
+#include <functional>
+
+namespace NYql::NDqs {
+
+void RegisterDqsMkqlCompilers(NCommon::TMkqlCallableCompilerBase& compiler, const TTypeAnnotationContext& ctx);
+
+}
diff --git a/ydb/library/yql/providers/dq/mkql/ya.make b/ydb/library/yql/providers/dq/mkql/ya.make
index 03ecd4352d..962ca294fe 100644
--- a/ydb/library/yql/providers/dq/mkql/ya.make
+++ b/ydb/library/yql/providers/dq/mkql/ya.make
@@ -1,20 +1,20 @@
-LIBRARY()
-
-OWNER(
- g:yql
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(
+ g:yql
+)
+
+PEERDIR(
ydb/library/yql/core
ydb/library/yql/providers/common/mkql
ydb/library/yql/providers/dq/expr_nodes
ydb/library/yql/providers/dq/interface
-)
-
-SRCS(
- dqs_mkql_compiler.cpp
-)
-
+)
+
+SRCS(
+ dqs_mkql_compiler.cpp
+)
+
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/library/yql/providers/dq/opt/dqs_opt.cpp b/ydb/library/yql/providers/dq/opt/dqs_opt.cpp
index b4bc07fbef..2b18f5b839 100644
--- a/ydb/library/yql/providers/dq/opt/dqs_opt.cpp
+++ b/ydb/library/yql/providers/dq/opt/dqs_opt.cpp
@@ -100,22 +100,22 @@ namespace NYql::NDqs {
}
THolder<IGraphTransformer> CreateDqsRewritePhyCallablesTransformer() {
- return CreateFunctorTransformer([](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- TOptimizeExprSettings optSettings{nullptr};
- optSettings.VisitLambdas = true;
- return OptimizeExprEx(input, output,
- [](const TExprNode::TPtr& inputExpr, TExprContext& ctx, IOptimizationContext&) {
- TExprBase node{inputExpr};
+ return CreateFunctorTransformer([](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ TOptimizeExprSettings optSettings{nullptr};
+ optSettings.VisitLambdas = true;
+ return OptimizeExprEx(input, output,
+ [](const TExprNode::TPtr& inputExpr, TExprContext& ctx, IOptimizationContext&) {
+ TExprBase node{inputExpr};
PERFORM_RULE(DqPeepholeRewriteCrossJoin, node, ctx);
PERFORM_RULE(DqPeepholeRewriteJoinDict, node, ctx);
- PERFORM_RULE(DqPeepholeRewriteMapJoin, node, ctx);
+ PERFORM_RULE(DqPeepholeRewriteMapJoin, node, ctx);
PERFORM_RULE(DqPeepholeRewritePureJoin, node, ctx);
PERFORM_RULE(DqPeepholeRewriteReplicate, node, ctx);
- return inputExpr;
- }, ctx, optSettings);
- });
- }
-
+ return inputExpr;
+ }, ctx, optSettings);
+ });
+ }
+
namespace NPeephole {
class TDqsPeepholeTransformer: public TSyncTransformerBase {
@@ -167,14 +167,14 @@ namespace NYql::NDqs {
return MakeHolder<NPeephole::TDqsPeepholeTransformer>(std::move(typeAnnTransformer), typesCtx);
}
- THolder<IGraphTransformer> CreateDqsFinalizingOptTransformer() {
- return CreateFunctorTransformer(
- [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ THolder<IGraphTransformer> CreateDqsFinalizingOptTransformer() {
+ return CreateFunctorTransformer(
+ [](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
auto status = NDq::DqReplicateStageMultiOutput(input, output, ctx);
- if (status.Level != TStatus::Error && input != output) {
- YQL_CVLOG(NLog::ELevel::INFO, NLog::EComponent::ProviderDq) << "DqReplicateStageMultiOutput";
- }
- return status;
+ if (status.Level != TStatus::Error && input != output) {
+ YQL_CVLOG(NLog::ELevel::INFO, NLog::EComponent::ProviderDq) << "DqReplicateStageMultiOutput";
+ }
+ return status;
});
}
}
diff --git a/ydb/library/yql/providers/dq/opt/dqs_opt.h b/ydb/library/yql/providers/dq/opt/dqs_opt.h
index 6c9e1a7c18..b8a9e93b88 100644
--- a/ydb/library/yql/providers/dq/opt/dqs_opt.h
+++ b/ydb/library/yql/providers/dq/opt/dqs_opt.h
@@ -8,7 +8,7 @@ namespace NYql::NDqs {
class TDatabaseManager;
THolder<IGraphTransformer> CreateDqsWrapListsOptTransformer();
- THolder<IGraphTransformer> CreateDqsFinalizingOptTransformer();
+ THolder<IGraphTransformer> CreateDqsFinalizingOptTransformer();
THolder<IGraphTransformer> CreateDqsBuildTransformer();
THolder<IGraphTransformer> CreateDqsRewritePhyCallablesTransformer();
THolder<IGraphTransformer> CreateDqsPeepholeTransformer(THolder<IGraphTransformer>&& typeAnnTransformer, TTypeAnnotationContext& typesCtx);
diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp
index 9e87f81e7c..30bd77f890 100644
--- a/ydb/library/yql/providers/dq/opt/logical_optimize.cpp
+++ b/ydb/library/yql/providers/dq/opt/logical_optimize.cpp
@@ -1,6 +1,6 @@
-#include "logical_optimize.h"
-#include "dqs_opt.h"
-
+#include "logical_optimize.h"
+#include "dqs_opt.h"
+
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
#include <ydb/library/yql/providers/common/transform/yql_optimize.h>
#include <ydb/library/yql/dq/opt/dq_opt_log.h>
@@ -8,13 +8,13 @@
#include <ydb/library/yql/dq/opt/dq_opt.h>
#include <ydb/library/yql/core/yql_opt_utils.h>
#include <ydb/library/yql/utils/log/log.h>
-
-namespace NYql::NDqs {
-
-using namespace NYql;
-using namespace NYql::NDq;
-using namespace NYql::NNodes;
-
+
+namespace NYql::NDqs {
+
+using namespace NYql;
+using namespace NYql::NDq;
+using namespace NYql::NNodes;
+
namespace {
constexpr TStringBuf YQL_TIME = "_yql_time";
@@ -39,41 +39,41 @@ TString BuildColumnName(const TExprBase column) {
}
-class TDqsLogicalOptProposalTransformer : public TOptimizeTransformerBase {
-public:
+class TDqsLogicalOptProposalTransformer : public TOptimizeTransformerBase {
+public:
TDqsLogicalOptProposalTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config)
- : TOptimizeTransformerBase(typeCtx, NLog::EComponent::ProviderDq, {})
+ : TOptimizeTransformerBase(typeCtx, NLog::EComponent::ProviderDq, {})
, Config(config)
- {
-#define HNDL(name) "DqsLogical-"#name, Hndl(&TDqsLogicalOptProposalTransformer::name)
+ {
+#define HNDL(name) "DqsLogical-"#name, Hndl(&TDqsLogicalOptProposalTransformer::name)
AddHandler(0, &TCoUnorderedBase::Match, HNDL(SkipUnordered));
- AddHandler(0, &TCoAggregate::Match, HNDL(RewriteAggregate));
- AddHandler(0, &TCoTake::Match, HNDL(RewriteTakeSortToTopSort));
- AddHandler(0, &TCoEquiJoin::Match, HNDL(RewriteEquiJoin));
+ AddHandler(0, &TCoAggregate::Match, HNDL(RewriteAggregate));
+ AddHandler(0, &TCoTake::Match, HNDL(RewriteTakeSortToTopSort));
+ AddHandler(0, &TCoEquiJoin::Match, HNDL(RewriteEquiJoin));
AddHandler(0, &TCoCalcOverWindow::Match, HNDL(ExpandWindowFunctions));
AddHandler(0, &TCoCalcOverWindowGroup::Match, HNDL(ExpandWindowFunctions));
AddHandler(0, &TCoFlatMapBase::Match, HNDL(FlatMapOverExtend));
AddHandler(0, &TDqQuery::Match, HNDL(MergeQueriesWithSinks));
- AddHandler(0, &TDqStageBase::Match, HNDL(UnorderedInStage));
-#undef HNDL
- }
-
-protected:
- TMaybeNode<TExprBase> SkipUnordered(TExprBase node, TExprContext& ctx) {
- Y_UNUSED(ctx);
+ AddHandler(0, &TDqStageBase::Match, HNDL(UnorderedInStage));
+#undef HNDL
+ }
+
+protected:
+ TMaybeNode<TExprBase> SkipUnordered(TExprBase node, TExprContext& ctx) {
+ Y_UNUSED(ctx);
auto unordered = node.Cast<TCoUnorderedBase>();
- if (unordered.Input().Maybe<TDqConnection>()) {
- return unordered.Input();
- }
- return node;
- }
-
+ if (unordered.Input().Maybe<TDqConnection>()) {
+ return unordered.Input();
+ }
+ return node;
+ }
+
TMaybeNode<TExprBase> FlatMapOverExtend(TExprBase node, TExprContext& ctx) {
return DqFlatMapOverExtend(node, ctx);
}
- TMaybeNode<TExprBase> RewriteAggregate(TExprBase node, TExprContext& ctx) {
- auto aggregate = node.Cast<TCoAggregate>();
+ TMaybeNode<TExprBase> RewriteAggregate(TExprBase node, TExprContext& ctx) {
+ auto aggregate = node.Cast<TCoAggregate>();
auto input = aggregate.Input().Maybe<TDqConnection>();
auto hopSetting = GetSetting(aggregate.Settings().Ref(), "hopping");
@@ -83,49 +83,49 @@ protected:
} else {
return DqRewriteAggregate(node, ctx);
}
- }
- return node;
- }
-
- TMaybeNode<TExprBase> RewriteTakeSortToTopSort(TExprBase node, TExprContext& ctx, const TGetParents& getParents) {
- if (node.Maybe<TCoTake>().Input().Maybe<TCoSort>().Input().Maybe<TDqConnection>()) {
- return DqRewriteTakeSortToTopSort(node, ctx, *getParents());
- }
- return node;
- }
-
- TMaybeNode<TExprBase> RewriteEquiJoin(TExprBase node, TExprContext& ctx) {
- auto equiJoin = node.Cast<TCoEquiJoin>();
- for (size_t i = 0; i + 2 < equiJoin.ArgCount(); ++i) {
- auto list = equiJoin.Arg(i).Cast<TCoEquiJoinInput>().List();
- if (auto maybeExtractMembers = list.Maybe<TCoExtractMembers>()) {
- list = maybeExtractMembers.Cast().Input();
- }
- if (auto maybeFlatMap = list.Maybe<TCoFlatMapBase>()) {
- list = maybeFlatMap.Cast().Input();
- }
- if (!list.Maybe<TDqConnection>()) {
- return node;
- }
- }
- return DqRewriteEquiJoin(node, ctx);
- }
+ }
+ return node;
+ }
+
+ TMaybeNode<TExprBase> RewriteTakeSortToTopSort(TExprBase node, TExprContext& ctx, const TGetParents& getParents) {
+ if (node.Maybe<TCoTake>().Input().Maybe<TCoSort>().Input().Maybe<TDqConnection>()) {
+ return DqRewriteTakeSortToTopSort(node, ctx, *getParents());
+ }
+ return node;
+ }
+
+ TMaybeNode<TExprBase> RewriteEquiJoin(TExprBase node, TExprContext& ctx) {
+ auto equiJoin = node.Cast<TCoEquiJoin>();
+ for (size_t i = 0; i + 2 < equiJoin.ArgCount(); ++i) {
+ auto list = equiJoin.Arg(i).Cast<TCoEquiJoinInput>().List();
+ if (auto maybeExtractMembers = list.Maybe<TCoExtractMembers>()) {
+ list = maybeExtractMembers.Cast().Input();
+ }
+ if (auto maybeFlatMap = list.Maybe<TCoFlatMapBase>()) {
+ list = maybeFlatMap.Cast().Input();
+ }
+ if (!list.Maybe<TDqConnection>()) {
+ return node;
+ }
+ }
+ return DqRewriteEquiJoin(node, ctx);
+ }
TMaybeNode<TExprBase> ExpandWindowFunctions(TExprBase node, TExprContext& ctx) {
- if (node.Cast<TCoInputBase>().Input().Maybe<TDqConnection>()) {
- return DqExpandWindowFunctions(node, ctx, true);
- }
- return node;
+ if (node.Cast<TCoInputBase>().Input().Maybe<TDqConnection>()) {
+ return DqExpandWindowFunctions(node, ctx, true);
+ }
+ return node;
}
TMaybeNode<TExprBase> MergeQueriesWithSinks(TExprBase node, TExprContext& ctx) {
return DqMergeQueriesWithSinks(node, ctx);
}
- TMaybeNode<TExprBase> UnorderedInStage(TExprBase node, TExprContext& ctx) const {
- return DqUnorderedInStage(node, TDqReadWrapBase::Match, ctx, Types);
- }
-
+ TMaybeNode<TExprBase> UnorderedInStage(TExprBase node, TExprContext& ctx) const {
+ return DqUnorderedInStage(node, TDqReadWrapBase::Match, ctx, Types);
+ }
+
private:
TMaybeNode<TExprBase> RewriteAsHoppingWindow(const TExprBase node, TExprContext& ctx, const TDqConnection& input) {
const auto aggregate = node.Cast<TCoAggregate>();
@@ -777,10 +777,10 @@ private:
private:
TDqConfiguration::TPtr Config;
-};
-
+};
+
THolder<IGraphTransformer> CreateDqsLogOptTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config) {
return THolder(new TDqsLogicalOptProposalTransformer(typeCtx, config));
-}
-
-} // NYql::NDqs
+}
+
+} // NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/opt/logical_optimize.h b/ydb/library/yql/providers/dq/opt/logical_optimize.h
index 54352ac344..03e5318aa5 100644
--- a/ydb/library/yql/providers/dq/opt/logical_optimize.h
+++ b/ydb/library/yql/providers/dq/opt/logical_optimize.h
@@ -1,13 +1,13 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/core/yql_graph_transformer.h>
#include <ydb/library/yql/core/yql_type_annotation.h>
#include <ydb/library/yql/providers/dq/common/yql_dq_settings.h>
-
-#include <util/generic/ptr.h>
-
-namespace NYql::NDqs {
-
+
+#include <util/generic/ptr.h>
+
+namespace NYql::NDqs {
+
THolder<IGraphTransformer> CreateDqsLogOptTransformer(TTypeAnnotationContext* typeCtx, const TDqConfiguration::TPtr& config);
-
-} // namespace NYql::NDqs
+
+} // namespace NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/opt/physical_optimize.cpp b/ydb/library/yql/providers/dq/opt/physical_optimize.cpp
index 858e3da45a..e7aedd1ae7 100644
--- a/ydb/library/yql/providers/dq/opt/physical_optimize.cpp
+++ b/ydb/library/yql/providers/dq/opt/physical_optimize.cpp
@@ -1,47 +1,47 @@
-#include "physical_optimize.h"
-
+#include "physical_optimize.h"
+
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
#include <ydb/library/yql/providers/common/transform/yql_optimize.h>
#include <ydb/library/yql/dq/opt/dq_opt_phy.h>
#include <ydb/library/yql/dq/opt/dq_opt.h>
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/core/yql_opt_utils.h>
-
+
#include <ydb/library/yql/utils/log/log.h>
-
-namespace NYql::NDqs {
-
-using namespace NYql;
-using namespace NYql::NDq;
-using namespace NYql::NNodes;
-
-class TDqsPhysicalOptProposalTransformer : public TOptimizeTransformerBase {
-public:
- TDqsPhysicalOptProposalTransformer(TTypeAnnotationContext* typeCtx)
- : TOptimizeTransformerBase(typeCtx, NLog::EComponent::ProviderDq, {})
- {
-#define HNDL(name) "DqsPhy-"#name, Hndl(&TDqsPhysicalOptProposalTransformer::name)
+
+namespace NYql::NDqs {
+
+using namespace NYql;
+using namespace NYql::NDq;
+using namespace NYql::NNodes;
+
+class TDqsPhysicalOptProposalTransformer : public TOptimizeTransformerBase {
+public:
+ TDqsPhysicalOptProposalTransformer(TTypeAnnotationContext* typeCtx)
+ : TOptimizeTransformerBase(typeCtx, NLog::EComponent::ProviderDq, {})
+ {
+#define HNDL(name) "DqsPhy-"#name, Hndl(&TDqsPhysicalOptProposalTransformer::name)
AddHandler(0, &TDqSourceWrap::Match, HNDL(BuildStageWithSourceWrap));
- AddHandler(0, &TDqReadWrap::Match, HNDL(BuildStageWithReadWrap));
- AddHandler(0, &TCoSkipNullMembers::Match, HNDL(PushSkipNullMembersToStage<false>));
- AddHandler(0, &TCoExtractMembers::Match, HNDL(PushExtractMembersToStage<false>));
+ AddHandler(0, &TDqReadWrap::Match, HNDL(BuildStageWithReadWrap));
+ AddHandler(0, &TCoSkipNullMembers::Match, HNDL(PushSkipNullMembersToStage<false>));
+ AddHandler(0, &TCoExtractMembers::Match, HNDL(PushExtractMembersToStage<false>));
AddHandler(0, &TCoFlatMapBase::Match, HNDL(BuildFlatmapStage<false>));
- AddHandler(0, &TCoCombineByKey::Match, HNDL(PushCombineToStage<false>));
- AddHandler(0, &TCoPartitionsByKeys::Match, HNDL(BuildPartitionsStage));
- AddHandler(0, &TCoPartitionByKey::Match, HNDL(BuildPartitionStage));
- AddHandler(0, &TCoAsList::Match, HNDL(BuildAggregationResultStage));
- AddHandler(0, &TCoTopSort::Match, HNDL(BuildTopSortStage<false>));
- AddHandler(0, &TCoSort::Match, HNDL(BuildSortStage<false>));
- AddHandler(0, &TCoTake::Match, HNDL(BuildTakeOrTakeSkipStage<false>));
- AddHandler(0, &TCoLength::Match, HNDL(RewriteLengthOfStageOutput));
+ AddHandler(0, &TCoCombineByKey::Match, HNDL(PushCombineToStage<false>));
+ AddHandler(0, &TCoPartitionsByKeys::Match, HNDL(BuildPartitionsStage));
+ AddHandler(0, &TCoPartitionByKey::Match, HNDL(BuildPartitionStage));
+ AddHandler(0, &TCoAsList::Match, HNDL(BuildAggregationResultStage));
+ AddHandler(0, &TCoTopSort::Match, HNDL(BuildTopSortStage<false>));
+ AddHandler(0, &TCoSort::Match, HNDL(BuildSortStage<false>));
+ AddHandler(0, &TCoTake::Match, HNDL(BuildTakeOrTakeSkipStage<false>));
+ AddHandler(0, &TCoLength::Match, HNDL(RewriteLengthOfStageOutput));
AddHandler(0, &TCoExtendBase::Match, HNDL(BuildExtendStage));
AddHandler(0, &TDqJoin::Match, HNDL(RewriteRightJoinToLeft));
- AddHandler(0, &TDqJoin::Match, HNDL(PushJoinToStage<false>));
- AddHandler(0, &TDqJoin::Match, HNDL(BuildJoin<false>));
+ AddHandler(0, &TDqJoin::Match, HNDL(PushJoinToStage<false>));
+ AddHandler(0, &TDqJoin::Match, HNDL(BuildJoin<false>));
AddHandler(0, &TDqJoin::Match, HNDL(BuildJoinDict<false>));
- AddHandler(0, &TCoAssumeSorted::Match, HNDL(BuildSortStage<false>));
- AddHandler(0, &TCoOrderedLMap::Match, HNDL(PushOrderedLMapToStage<false>));
- AddHandler(0, &TCoLMap::Match, HNDL(PushLMapToStage<false>));
+ AddHandler(0, &TCoAssumeSorted::Match, HNDL(BuildSortStage<false>));
+ AddHandler(0, &TCoOrderedLMap::Match, HNDL(PushOrderedLMapToStage<false>));
+ AddHandler(0, &TCoLMap::Match, HNDL(PushLMapToStage<false>));
// (Apply (SqlExternalFunction ..) ..) to stage
AddHandler(0, &TCoApply::Match, HNDL(BuildExtFunctionStage<false>));
#if 0
@@ -49,26 +49,26 @@ public:
AddHandler(0, &TCoToOptional::Match, HNDL(BuildScalarPrecompute));
#endif
- AddHandler(1, &TCoSkipNullMembers::Match, HNDL(PushSkipNullMembersToStage<true>));
- AddHandler(1, &TCoExtractMembers::Match, HNDL(PushExtractMembersToStage<true>));
+ AddHandler(1, &TCoSkipNullMembers::Match, HNDL(PushSkipNullMembersToStage<true>));
+ AddHandler(1, &TCoExtractMembers::Match, HNDL(PushExtractMembersToStage<true>));
AddHandler(1, &TCoFlatMapBase::Match, HNDL(BuildFlatmapStage<true>));
- AddHandler(1, &TCoCombineByKey::Match, HNDL(PushCombineToStage<true>));
- AddHandler(1, &TCoTopSort::Match, HNDL(BuildTopSortStage<true>));
- AddHandler(1, &TCoSort::Match, HNDL(BuildSortStage<true>));
- AddHandler(1, &TCoTake::Match, HNDL(BuildTakeOrTakeSkipStage<true>));
- AddHandler(1, &TDqJoin::Match, HNDL(PushJoinToStage<true>));
- AddHandler(1, &TDqJoin::Match, HNDL(BuildJoin<true>));
+ AddHandler(1, &TCoCombineByKey::Match, HNDL(PushCombineToStage<true>));
+ AddHandler(1, &TCoTopSort::Match, HNDL(BuildTopSortStage<true>));
+ AddHandler(1, &TCoSort::Match, HNDL(BuildSortStage<true>));
+ AddHandler(1, &TCoTake::Match, HNDL(BuildTakeOrTakeSkipStage<true>));
+ AddHandler(1, &TDqJoin::Match, HNDL(PushJoinToStage<true>));
+ AddHandler(1, &TDqJoin::Match, HNDL(BuildJoin<true>));
AddHandler(1, &TDqJoin::Match, HNDL(BuildJoinDict<true>));
- AddHandler(1, &TCoAssumeSorted::Match, HNDL(BuildSortStage<true>));
- AddHandler(1, &TCoOrderedLMap::Match, HNDL(PushOrderedLMapToStage<true>));
- AddHandler(1, &TCoLMap::Match, HNDL(PushLMapToStage<true>));
+ AddHandler(1, &TCoAssumeSorted::Match, HNDL(BuildSortStage<true>));
+ AddHandler(1, &TCoOrderedLMap::Match, HNDL(PushOrderedLMapToStage<true>));
+ AddHandler(1, &TCoLMap::Match, HNDL(PushLMapToStage<true>));
AddHandler(1, &TCoApply::Match, HNDL(BuildExtFunctionStage<true>));
-#undef HNDL
-
- SetGlobal(1u);
- }
-
-protected:
+#undef HNDL
+
+ SetGlobal(1u);
+ }
+
+protected:
TMaybeNode<TExprBase> BuildStageWithSourceWrap(TExprBase node, TExprContext& ctx) {
const auto wrap = node.Cast<TDqSourceWrap>();
const auto& items = GetSeqItemType(wrap.Ref().GetTypeAnn())->Cast<TStructExprType>()->GetItems();
@@ -145,13 +145,13 @@ protected:
.Build().Done();
}
- TMaybeNode<TExprBase> BuildStageWithReadWrap(TExprBase node, TExprContext& ctx) {
+ TMaybeNode<TExprBase> BuildStageWithReadWrap(TExprBase node, TExprContext& ctx) {
const auto wrap = node.Cast<TDqReadWrap>();
const auto read = Build<TDqReadWideWrap>(ctx, node.Pos())
.Input(wrap.Input())
.Token(wrap.Token())
.Done();
-
+
const auto structType = GetSeqItemType(wrap.Ref().GetTypeAnn())->Cast<TStructExprType>();
auto narrow = ctx.Builder(node.Pos())
.Lambda()
@@ -176,107 +176,107 @@ protected:
.Seal()
.Seal().Build();
- return Build<TDqCnUnionAll>(ctx, node.Pos())
- .Output()
+ return Build<TDqCnUnionAll>(ctx, node.Pos())
+ .Output()
.Stage<TDqStage>()
.Inputs().Build()
.Program(narrow)
.Settings(TDqStageSettings().BuildNode(ctx, node.Pos()))
.Build()
- .Index().Build("0")
+ .Index().Build("0")
.Build() .Done();
- }
-
- template <bool IsGlobal>
- TMaybeNode<TExprBase> PushSkipNullMembersToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
+ }
+
+ template <bool IsGlobal>
+ TMaybeNode<TExprBase> PushSkipNullMembersToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
return DqPushSkipNullMembersToStage(node, ctx, optCtx, *getParents(), IsGlobal);
- }
-
- template <bool IsGlobal>
- TMaybeNode<TExprBase> PushExtractMembersToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
+ }
+
+ template <bool IsGlobal>
+ TMaybeNode<TExprBase> PushExtractMembersToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
return DqPushExtractMembersToStage(node, ctx, optCtx, *getParents(), IsGlobal);
- }
-
- template <bool IsGlobal>
+ }
+
+ template <bool IsGlobal>
TMaybeNode<TExprBase> BuildFlatmapStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
return DqBuildFlatmapStage(node, ctx, optCtx, *getParents(), IsGlobal);
- }
-
- template <bool IsGlobal>
+ }
+
+ template <bool IsGlobal>
TMaybeNode<TExprBase> PushOrderedLMapToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
return DqPushOrderedLMapToStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
- template <bool IsGlobal>
+ template <bool IsGlobal>
TMaybeNode<TExprBase> PushLMapToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
return DqPushLMapToStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
- template <bool IsGlobal>
+ template <bool IsGlobal>
TMaybeNode<TExprBase> BuildExtFunctionStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
return DqBuildExtFunctionStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
template <bool IsGlobal>
- TMaybeNode<TExprBase> PushCombineToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
+ TMaybeNode<TExprBase> PushCombineToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
return DqPushCombineToStage(node, ctx, optCtx, *getParents(), IsGlobal);
- }
-
- TMaybeNode<TExprBase> BuildPartitionsStage(TExprBase node, TExprContext& ctx, const TGetParents& getParents) {
- return DqBuildPartitionsStage(node, ctx, *getParents());
- }
-
- TMaybeNode<TExprBase> BuildPartitionStage(TExprBase node, TExprContext& ctx, const TGetParents& getParents) {
- return DqBuildPartitionStage(node, ctx, *getParents());
- }
-
- TMaybeNode<TExprBase> BuildAggregationResultStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
- return DqBuildAggregationResultStage(node, ctx, optCtx);
- }
-
- template <bool IsGlobal>
- TMaybeNode<TExprBase> BuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
+ }
+
+ TMaybeNode<TExprBase> BuildPartitionsStage(TExprBase node, TExprContext& ctx, const TGetParents& getParents) {
+ return DqBuildPartitionsStage(node, ctx, *getParents());
+ }
+
+ TMaybeNode<TExprBase> BuildPartitionStage(TExprBase node, TExprContext& ctx, const TGetParents& getParents) {
+ return DqBuildPartitionStage(node, ctx, *getParents());
+ }
+
+ TMaybeNode<TExprBase> BuildAggregationResultStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
+ return DqBuildAggregationResultStage(node, ctx, optCtx);
+ }
+
+ template <bool IsGlobal>
+ TMaybeNode<TExprBase> BuildTopSortStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
return DqBuildTopSortStage(node, ctx, optCtx, *getParents(), IsGlobal);
- }
-
- template <bool IsGlobal>
- TMaybeNode<TExprBase> BuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
+ }
+
+ template <bool IsGlobal>
+ TMaybeNode<TExprBase> BuildSortStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
return DqBuildSortStage(node, ctx, optCtx, *getParents(), IsGlobal);
- }
-
- template <bool IsGlobal>
+ }
+
+ template <bool IsGlobal>
TMaybeNode<TExprBase> BuildTakeOrTakeSkipStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
if (node.Maybe<TCoTake>().Input().Maybe<TCoSkip>()) {
return DqBuildTakeSkipStage(node, ctx, optCtx, *getParents(), IsGlobal);
} else {
return DqBuildTakeStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
- }
-
- TMaybeNode<TExprBase> RewriteLengthOfStageOutput(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
- return DqRewriteLengthOfStageOutput(node, ctx, optCtx);
- }
-
- TMaybeNode<TExprBase> BuildExtendStage(TExprBase node, TExprContext& ctx) {
+ }
+
+ TMaybeNode<TExprBase> RewriteLengthOfStageOutput(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
+ return DqRewriteLengthOfStageOutput(node, ctx, optCtx);
+ }
+
+ TMaybeNode<TExprBase> BuildExtendStage(TExprBase node, TExprContext& ctx) {
return DqBuildExtendStage(node, ctx);
- }
-
+ }
+
TMaybeNode<TExprBase> RewriteRightJoinToLeft(TExprBase node, TExprContext& ctx) {
return DqRewriteRightJoinToLeft(node, ctx);
}
- template <bool IsGlobal>
+ template <bool IsGlobal>
TMaybeNode<TExprBase> PushJoinToStage(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
return DqPushJoinToStage(node, ctx, optCtx, *getParents(), IsGlobal);
}
- template <bool IsGlobal>
- TMaybeNode<TExprBase> BuildJoin(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
- auto join = node.Cast<TDqJoin>();
+ template <bool IsGlobal>
+ TMaybeNode<TExprBase> BuildJoin(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx, const TGetParents& getParents) {
+ auto join = node.Cast<TDqJoin>();
const TParentsMap* parentsMap = getParents();
if (!JoinPrerequisitesVerify(join, parentsMap, IsGlobal)) {
- return node;
- }
+ return node;
+ }
return DqBuildPhyJoin(join, false /* TODO */, ctx, optCtx);
}
@@ -286,14 +286,14 @@ protected:
Y_UNUSED(optCtx);
auto join = node.Cast<TDqJoin>();
- const TParentsMap* parentsMap = getParents();
+ const TParentsMap* parentsMap = getParents();
if (!JoinPrerequisitesVerify(join, parentsMap, IsGlobal)) {
- return node;
- }
+ return node;
+ }
return DqBuildJoinDict(join, ctx); // , optCtx);
- }
-
+ }
+
TMaybeNode<TExprBase> BuildHasItems(TExprBase node, TExprContext& ctx, IOptimizationContext& optCtx) {
return DqBuildHasItems(node, ctx, optCtx);
}
@@ -320,10 +320,10 @@ private:
}
return true;
}
-};
-
-THolder<IGraphTransformer> CreateDqsPhyOptTransformer(TTypeAnnotationContext* typeCtx) {
+};
+
+THolder<IGraphTransformer> CreateDqsPhyOptTransformer(TTypeAnnotationContext* typeCtx) {
return THolder(new TDqsPhysicalOptProposalTransformer(typeCtx));
-}
-
-} // NYql::NDqs
+}
+
+} // NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/opt/physical_optimize.h b/ydb/library/yql/providers/dq/opt/physical_optimize.h
index 7ead367cbc..d5767527f4 100644
--- a/ydb/library/yql/providers/dq/opt/physical_optimize.h
+++ b/ydb/library/yql/providers/dq/opt/physical_optimize.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/core/yql_graph_transformer.h>
#include <ydb/library/yql/core/yql_type_annotation.h>
-
-#include <util/generic/ptr.h>
-
-namespace NYql::NDqs {
-
-THolder<IGraphTransformer> CreateDqsPhyOptTransformer(TTypeAnnotationContext* typeCtx);
-
-} // namespace NYql::NDqs
+
+#include <util/generic/ptr.h>
+
+namespace NYql::NDqs {
+
+THolder<IGraphTransformer> CreateDqsPhyOptTransformer(TTypeAnnotationContext* typeCtx);
+
+} // namespace NYql::NDqs
diff --git a/ydb/library/yql/providers/dq/opt/ya.make b/ydb/library/yql/providers/dq/opt/ya.make
index a4f1eccd39..c9c18416b4 100644
--- a/ydb/library/yql/providers/dq/opt/ya.make
+++ b/ydb/library/yql/providers/dq/opt/ya.make
@@ -12,8 +12,8 @@ PEERDIR(
SRCS(
dqs_opt.cpp
- logical_optimize.cpp
- physical_optimize.cpp
+ logical_optimize.cpp
+ physical_optimize.cpp
)
YQL_LAST_ABI_VERSION()
diff --git a/ydb/library/yql/providers/dq/planner/dqs_task_graph.h b/ydb/library/yql/providers/dq/planner/dqs_task_graph.h
index 0b62dfce4a..3d0d8df531 100644
--- a/ydb/library/yql/providers/dq/planner/dqs_task_graph.h
+++ b/ydb/library/yql/providers/dq/planner/dqs_task_graph.h
@@ -15,10 +15,10 @@ namespace NYql::NDqs {
struct TTaskOutputMeta {};
- struct TTaskMeta {
- THashMap<TString, TString> TaskParams;
+ struct TTaskMeta {
+ THashMap<TString, TString> TaskParams;
TString ClusterNameHint;
- };
+ };
using TStageInfo = NYql::NDq::TStageInfo<TStageInfoMeta>;
using TTaskOutput = NYql::NDq::TTaskOutput<TTaskOutputMeta>;
diff --git a/ydb/library/yql/providers/dq/planner/execution_planner.cpp b/ydb/library/yql/providers/dq/planner/execution_planner.cpp
index ec404c0a6d..c6f104d084 100644
--- a/ydb/library/yql/providers/dq/planner/execution_planner.cpp
+++ b/ydb/library/yql/providers/dq/planner/execution_planner.cpp
@@ -60,15 +60,15 @@ namespace NYql::NDqs {
return stages;
}
- bool HasReadWraps(const TExprNode::TPtr& node) {
- bool result = false;
- VisitExpr(node, [&result](const TExprNode::TPtr& exprNode) {
+ bool HasReadWraps(const TExprNode::TPtr& node) {
+ bool result = false;
+ VisitExpr(node, [&result](const TExprNode::TPtr& exprNode) {
result |= TMaybeNode<TDqReadWrapBase>(exprNode).IsValid();
- return !result;
- });
-
- return result;
- }
+ return !result;
+ });
+
+ return result;
+ }
static bool HasDqSource(const TDqPhyStage& stage) {
for (size_t inputIndex = 0; inputIndex < stage.Inputs().Size(); ++inputIndex) {
@@ -84,13 +84,13 @@ namespace NYql::NDqs {
TDqsExecutionPlanner::TDqsExecutionPlanner(TIntrusivePtr<TTypeAnnotationContext> typeContext,
NYql::TExprContext& exprContext,
const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- NYql::TExprNode::TPtr dqExprRoot,
- NActors::TActorId executerID,
+ NYql::TExprNode::TPtr dqExprRoot,
+ NActors::TActorId executerID,
NActors::TActorId resultID)
: TypeContext(std::move(typeContext))
, ExprContext(exprContext)
, FunctionRegistry(functionRegistry)
- , DqExprRoot(std::move(dqExprRoot))
+ , DqExprRoot(std::move(dqExprRoot))
, ExecuterID(executerID)
, ResultID(resultID)
{
@@ -106,7 +106,7 @@ namespace NYql::NDqs {
bool flag = true;
VisitExpr(
- DqExprRoot,
+ DqExprRoot,
[&](const TExprNode::TPtr& exprNode) {
if (exprNode->IsCallable("DataSource")) {
sources.insert(TString{exprNode->Child(0)->Content()});
@@ -125,19 +125,19 @@ namespace NYql::NDqs {
}
ui64 TDqsExecutionPlanner::StagesCount() {
- auto stages = GetStages(DqExprRoot);
+ auto stages = GetStages(DqExprRoot);
return stages.size();
}
- ui32 TDqsExecutionPlanner::PlanExecution(const TDqSettings::TPtr& settings, bool canFallback) {
- TExprBase expr(DqExprRoot);
+ ui32 TDqsExecutionPlanner::PlanExecution(const TDqSettings::TPtr& settings, bool canFallback) {
+ TExprBase expr(DqExprRoot);
auto result = expr.Maybe<TDqCnResult>();
auto query = expr.Maybe<TDqQuery>();
const auto maxTasksPerOperation = settings->MaxTasksPerOperation.Get().GetOrElse(TDqSettings::TDefault::MaxTasksPerOperation);
- YQL_LOG(DEBUG) << "Execution Plan " << NCommon::ExprToPrettyString(ExprContext, *DqExprRoot);
+ YQL_LOG(DEBUG) << "Execution Plan " << NCommon::ExprToPrettyString(ExprContext, *DqExprRoot);
- auto stages = GetStages(DqExprRoot);
+ auto stages = GetStages(DqExprRoot);
YQL_ENSURE(!stages.empty());
for (const auto& stage : stages) {
@@ -146,7 +146,7 @@ namespace NYql::NDqs {
for (const auto& stage : stages) {
const bool hasDqSource = HasDqSource(stage);
- if ((hasDqSource || HasReadWraps(stage.Program().Ptr())) && BuildReadStage(settings, stage, hasDqSource, canFallback)) {
+ if ((hasDqSource || HasReadWraps(stage.Program().Ptr())) && BuildReadStage(settings, stage, hasDqSource, canFallback)) {
YQL_LOG(DEBUG) << "Read stage " << NCommon::ExprToPrettyString(ExprContext, *stage.Ptr());
} else {
YQL_LOG(DEBUG) << "Common stage " << NCommon::ExprToPrettyString(ExprContext, *stage.Ptr());
@@ -185,7 +185,7 @@ namespace NYql::NDqs {
BuildConnections(stage);
- if (canFallback && TasksGraph.GetTasks().size() > maxTasksPerOperation) {
+ if (canFallback && TasksGraph.GetTasks().size() > maxTasksPerOperation) {
break;
}
}
@@ -335,9 +335,9 @@ namespace NYql::NDqs {
taskDesc.SetId(task.Id);
NActors::ActorIdToProto(ExecuterID, taskDesc.MutableExecuter()->MutableActorId());
auto& taskParams = *taskMeta.MutableTaskParams();
- for (const auto& [k, v]: task.Meta.TaskParams) {
- taskParams[k] = v;
- }
+ for (const auto& [k, v]: task.Meta.TaskParams) {
+ taskParams[k] = v;
+ }
if (clusterNameHints.size() == 1) {
taskMeta.SetClusterNameHint(*clusterNameHints.begin());
} else {
@@ -408,7 +408,7 @@ namespace NYql::NDqs {
return {};
}
- bool TDqsExecutionPlanner::BuildReadStage(const TDqSettings::TPtr& settings, const TDqPhyStage& stage, bool dqSource, bool canFallback) {
+ bool TDqsExecutionPlanner::BuildReadStage(const TDqSettings::TPtr& settings, const TDqPhyStage& stage, bool dqSource, bool canFallback) {
auto& stageInfo = TasksGraph.GetStageInfo(stage);
for (ui32 i = 0; i < stageInfo.InputsCount; i++) {
@@ -426,7 +426,7 @@ namespace NYql::NDqs {
dqSourceInputIndex = i;
break;
}
- }
+ }
YQL_ENSURE(dqSourceInputIndex < stageInfo.InputsCount);
} else {
if (const auto& wrap = FindNode(stage.Program().Ptr(), [](const TExprNode::TPtr& exprNode) {
@@ -462,7 +462,7 @@ namespace NYql::NDqs {
TVector<TString> parts;
if (auto dqIntegration = (*datasource)->GetDqIntegration()) {
TString clusterName;
- _MaxDataSizePerJob = Max(_MaxDataSizePerJob, dqIntegration->Partition(*settings, maxPartitions, *read, parts, &clusterName, ExprContext, canFallback));
+ _MaxDataSizePerJob = Max(_MaxDataSizePerJob, dqIntegration->Partition(*settings, maxPartitions, *read, parts, &clusterName, ExprContext, canFallback));
TMaybe<::google::protobuf::Any> sourceSettings;
TString sourceType;
if (dqSource) {
@@ -472,7 +472,7 @@ namespace NYql::NDqs {
YQL_ENSURE(sourceType, "Data source provider \"" << dataSourceName << "\" did't fill dq source settings type for its dq source node");
}
for (const auto& p : parts) {
- auto& task = TasksGraph.AddTask(stageInfo);
+ auto& task = TasksGraph.AddTask(stageInfo);
task.Meta.TaskParams[dataSourceName] = p;
task.Meta.ClusterNameHint = clusterName;
if (dqSource) {
@@ -482,7 +482,7 @@ namespace NYql::NDqs {
auto& transform = task.OutputTransform;
transform.Type = stageSettings.TransformType;
transform.FunctionName = stageSettings.TransformName;
- }
+ }
}
return !parts.empty();
}
@@ -515,13 +515,13 @@ namespace NYql::NDqs {
THashMap<TStageId, std::tuple<TString,ui64>> TDqsExecutionPlanner::BuildAllPrograms() {
using namespace NKikimr::NMiniKQL;
-
+
THashMap<TStageId, std::tuple<TString,ui64>> result;
TScopedAlloc alloc(NKikimr::TAlignedPagePoolCounters(), FunctionRegistry->SupportsSizedAllocators());
TTypeEnvironment typeEnv(alloc);
- TVector<NNodes::TExprBase> fakeReads;
+ TVector<NNodes::TExprBase> fakeReads;
NCommon::TMkqlCommonCallableCompiler compiler;
- RegisterDqsMkqlCompilers(compiler, *TypeContext);
+ RegisterDqsMkqlCompilers(compiler, *TypeContext);
for (const auto& stageInfo : TasksGraph.GetStagesInfo()) {
const auto& stage = stageInfo.second.Meta.Stage;
@@ -602,7 +602,7 @@ namespace NYql::NDqs {
default:
YQL_ENSURE(false, "Unexpected task input type.");
}
-
+
for (ui64 channel : input.Channels) {
auto& channelDesc = *inputDesc.AddChannels();
FillChannelDesc(channelDesc, TasksGraph.GetChannel(channel));
diff --git a/ydb/library/yql/providers/dq/planner/execution_planner.h b/ydb/library/yql/providers/dq/planner/execution_planner.h
index 280600c8f7..f8212c8f50 100644
--- a/ydb/library/yql/providers/dq/planner/execution_planner.h
+++ b/ydb/library/yql/providers/dq/planner/execution_planner.h
@@ -11,8 +11,8 @@
#include <util/generic/vector.h>
-#include <tuple>
-
+#include <tuple>
+
namespace NYql::NDqs {
class IDqsExecutionPlanner {
public:
@@ -28,7 +28,7 @@ namespace NYql::NDqs {
explicit TDqsExecutionPlanner(TIntrusivePtr<TTypeAnnotationContext> typeContext,
NYql::TExprContext& exprContext,
const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- NYql::TExprNode::TPtr dqExprRoot,
+ NYql::TExprNode::TPtr dqExprRoot,
NActors::TActorId executerID = NActors::TActorId(),
NActors::TActorId resultID = NActors::TActorId(1, 0, 1, 0));
@@ -38,7 +38,7 @@ namespace NYql::NDqs {
return _MaxDataSizePerJob;
}
ui64 StagesCount();
- ui32 PlanExecution(const TDqSettings::TPtr& settings, bool canFallback = false);
+ ui32 PlanExecution(const TDqSettings::TPtr& settings, bool canFallback = false);
TVector<NDqProto::TDqTask> GetTasks(const TVector<NActors::TActorId>& workers) override;
TVector<NDqProto::TDqTask>& GetTasks() override;
@@ -50,22 +50,22 @@ namespace NYql::NDqs {
}
private:
- bool BuildReadStage(const TDqSettings::TPtr& settings, const NNodes::TDqPhyStage& stage, bool dqSource, bool canFallback);
+ bool BuildReadStage(const TDqSettings::TPtr& settings, const NNodes::TDqPhyStage& stage, bool dqSource, bool canFallback);
void BuildConnections(const NNodes::TDqPhyStage& stage);
THashMap<NDq::TStageId, std::tuple<TString,ui64>> BuildAllPrograms();
void FillChannelDesc(NDqProto::TChannel& channelDesc, const NDq::TChannel& channel);
void FillInputDesc(NDqProto::TTaskInput& inputDesc, const TTaskInput& input);
void FillOutputDesc(NDqProto::TTaskOutput& outputDesc, const TTaskOutput& output);
- void GatherPhyMapping(THashMap<std::tuple<TString, TString>, TString>& clusters, THashMap<std::tuple<TString, TString, TString>, TString>& tables);
+ void GatherPhyMapping(THashMap<std::tuple<TString, TString>, TString>& clusters, THashMap<std::tuple<TString, TString, TString>, TString>& tables);
void BuildCheckpointingMode();
bool IsEgressTask(const TDqsTasksGraph::TTaskType& task) const;
-
+
private:
TIntrusivePtr<TTypeAnnotationContext> TypeContext;
NYql::TExprContext& ExprContext;
const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry;
- NYql::TExprNode::TPtr DqExprRoot;
+ NYql::TExprNode::TPtr DqExprRoot;
TVector<const TTypeAnnotationNode*> InputType;
NActors::TActorId ExecuterID;
NActors::TActorId ResultID;
diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
index 8759f44221..ae73129321 100644
--- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
+++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.cpp
@@ -48,10 +48,10 @@
#include <library/cpp/digest/md5/md5.h>
#include <util/system/env.h>
-#include <util/generic/size_literals.h>
-#include <util/stream/file.h>
-#include <util/string/builder.h>
-
+#include <util/generic/size_literals.h>
+#include <util/stream/file.h>
+#include <util/string/builder.h>
+
namespace NYql {
using namespace NCommon;
@@ -75,16 +75,16 @@ public:
{
try {
return ExecuteUnsafe(lambda, columns, secureParams, fillSettings);
- } catch (const NKikimr::TMemoryLimitExceededException& e) {
- auto res = ResultFromError<IDqGateway::TResult>(TStringBuilder()
- << "DQ computation exceeds the memory limit " << State->Settings->MemoryLimit.Get().GetOrElse(0) << ". Try to increase the limit using PRAGMA dq.MemoryLimit", pos);
- return NThreading::MakeFuture(res);
- } catch (const std::exception& e) {
- return NThreading::MakeFuture(ResultFromException<IDqGateway::TResult>(e, pos));
+ } catch (const NKikimr::TMemoryLimitExceededException& e) {
+ auto res = ResultFromError<IDqGateway::TResult>(TStringBuilder()
+ << "DQ computation exceeds the memory limit " << State->Settings->MemoryLimit.Get().GetOrElse(0) << ". Try to increase the limit using PRAGMA dq.MemoryLimit", pos);
+ return NThreading::MakeFuture(res);
+ } catch (const std::exception& e) {
+ return NThreading::MakeFuture(ResultFromException<IDqGateway::TResult>(e, pos));
} catch (...) {
- auto res = ResultFromError<IDqGateway::TResult>(CurrentExceptionMessage(), pos);
- res.SetStatus(TIssuesIds::UNEXPECTED);
- return NThreading::MakeFuture(res);
+ auto res = ResultFromError<IDqGateway::TResult>(CurrentExceptionMessage(), pos);
+ res.SetStatus(TIssuesIds::UNEXPECTED);
+ return NThreading::MakeFuture(res);
}
}
@@ -121,16 +121,16 @@ public:
NDq::TDqTaskRunnerContext executionContext;
executionContext.FuncRegistry = State->FunctionRegistry;
- executionContext.ComputationFactory = State->ComputationFactory;
+ executionContext.ComputationFactory = State->ComputationFactory;
executionContext.RandomProvider = randomProvider.Get();
executionContext.TimeProvider = timeProvider.Get();
NDq::TDqTaskRunnerMemoryLimits limits;
- limits.ChannelBufferSize = 10_MB;
- limits.OutputChunkMaxSize = 2_MB;
+ limits.ChannelBufferSize = 10_MB;
+ limits.OutputChunkMaxSize = 2_MB;
NDq::TDqTaskRunnerSettings settings;
- settings.OptLLVM = "OFF"; // Don't use LLVM for local execution
+ settings.OptLLVM = "OFF"; // Don't use LLVM for local execution
settings.SecureParams = secureParams;
settings.CollectBasicStats = true;
settings.CollectProfileStats = true;
@@ -150,21 +150,21 @@ public:
NDq::ERunStatus status;
while ((status = runner->Run()) == NDq::ERunStatus::PendingOutput || status == NDq::ERunStatus::Finished) {
NDqProto::TData data;
- if (runner->GetOutputChannel(0)->PopAll(data) && !fillSettings.Discard) {
+ if (runner->GetOutputChannel(0)->PopAll(data) && !fillSettings.Discard) {
rows.push_back(data);
}
if (status == NDq::ERunStatus::Finished) {
break;
}
- if (!fillSettings.Discard) {
- if (fillSettings.AllResultsBytesLimit && runner->GetOutputChannel(0)->GetStats()->Bytes >= *fillSettings.AllResultsBytesLimit) {
- result.Truncated = true;
- break;
- }
- if (fillSettings.RowsLimitPerWrite && runner->GetOutputChannel(0)->GetStats()->RowsOut >= *fillSettings.RowsLimitPerWrite) {
- result.Truncated = true;
- break;
- }
+ if (!fillSettings.Discard) {
+ if (fillSettings.AllResultsBytesLimit && runner->GetOutputChannel(0)->GetStats()->Bytes >= *fillSettings.AllResultsBytesLimit) {
+ result.Truncated = true;
+ break;
+ }
+ if (fillSettings.RowsLimitPerWrite && runner->GetOutputChannel(0)->GetStats()->RowsOut >= *fillSettings.RowsLimitPerWrite) {
+ result.Truncated = true;
+ break;
+ }
}
}
@@ -562,12 +562,12 @@ private:
}
}
- static TStatus FallbackCallback(const TDqStatePtr& state, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx)
+ static TStatus FallbackCallback(const TDqStatePtr& state, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx)
{
- if (state->Metrics) {
- state->Metrics->IncCounter("dq", "Fallback");
+ if (state->Metrics) {
+ state->Metrics->IncCounter("dq", "Fallback");
}
- state->Statistics[state->MetricId++].Entries.push_back(TOperationStatistics::TEntry("Fallback", 0, 0, 0, 0, 1));
+ state->Statistics[state->MetricId++].Entries.push_back(TOperationStatistics::TEntry("Fallback", 0, 0, 0, 0, 1));
YQL_ENSURE(input->ChildrenSize() > 0, "Node: " << NCommon::ExprToPrettyString(ctx, *input));
TExprNode::TPtr resFill = input->TailPtr();
@@ -582,10 +582,10 @@ private:
}
YQL_ENSURE(inMemoryIndex != resFill->ChildrenSize(), "Node: " << NCommon::ExprToPrettyString(ctx, *input));
- YQL_ENSURE(!state->TypeCtx->AvailablePureResultDataSources.empty());
- YQL_ENSURE(state->TypeCtx->AvailablePureResultDataSources.front() != DqProviderName);
+ YQL_ENSURE(!state->TypeCtx->AvailablePureResultDataSources.empty());
+ YQL_ENSURE(state->TypeCtx->AvailablePureResultDataSources.front() != DqProviderName);
- auto newAtom = ctx.NewAtom(input->Pos(), state->TypeCtx->AvailablePureResultDataSources.front());
+ auto newAtom = ctx.NewAtom(input->Pos(), state->TypeCtx->AvailablePureResultDataSources.front());
resFill->Child(inMemoryIndex)->SetState(TExprNode::EState::ExecutionComplete);
resFill = ctx.ChangeChild(*resFill, inMemoryIndex, std::move(newAtom));
@@ -593,14 +593,14 @@ private:
output = ctx.ChangeChild(*input, input->ChildrenSize()-1, std::move(resFill));
return TStatus::Repeat;
} else {
- YQL_ENSURE(!state->TypeCtx->AvailablePureResultDataSources.empty());
- YQL_ENSURE(state->TypeCtx->AvailablePureResultDataSources.front() != DqProviderName);
+ YQL_ENSURE(!state->TypeCtx->AvailablePureResultDataSources.empty());
+ YQL_ENSURE(state->TypeCtx->AvailablePureResultDataSources.front() != DqProviderName);
TStringStream out;
NYson::TYsonWriter writer((IOutputStream*)&out);
writer.OnBeginMap();
writer.OnKeyedItem("FallbackProvider");
- writer.OnRaw(state->TypeCtx->AvailablePureResultDataSources.front());
+ writer.OnRaw(state->TypeCtx->AvailablePureResultDataSources.front());
writer.OnEndMap();
output->SetResult(ctx.NewAtom(input->Pos(), out.Str()));
@@ -609,8 +609,8 @@ private:
}
TStatusCallbackPair Fallback() const {
- auto callback = TAsyncTransformCallback([state = State] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- return FallbackCallback(state, input, output, ctx);
+ auto callback = TAsyncTransformCallback([state = State] (const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ return FallbackCallback(state, input, output, ctx);
});
return std::make_pair(TStatus::Async, NThreading::MakeFuture(callback));
}
@@ -628,9 +628,9 @@ private:
auto result = TMaybeNode<TResult>(input).Cast();
IDataProvider::TFillSettings fillSettings = NCommon::GetFillSettings(result.Ref());
auto settings = State->Settings->WithFillSettings(fillSettings);
- if (!settings->_RowsLimitPerWrite.Get() && !settings->_AllResultsBytesLimit.Get()) {
- settings->_AllResultsBytesLimit = 64_MB;
- }
+ if (!settings->_RowsLimitPerWrite.Get() && !settings->_AllResultsBytesLimit.Get()) {
+ settings->_AllResultsBytesLimit = 64_MB;
+ }
THashMap<TString, TString> secureParams;
NCommon::FillSecureParams(result.Input().Ptr(), *State->TypeCtx, secureParams);
@@ -683,16 +683,16 @@ private:
}
tasks[0].MutableMeta()->PackFrom(taskMeta);
- bool enableFullResultWrite = settings->EnableFullResultWrite.Get().GetOrElse(false);
- if (enableFullResultWrite) {
- const auto type = result.Input().Ref().GetTypeAnn();
- const auto integration = GetDqIntegrationForFullResTable(State);
- enableFullResultWrite = type->GetKind() == ETypeAnnotationKind::List
- && type->Cast<TListExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Struct
- && !fillSettings.Discard
- && integration
- && integration->PrepareFullResultTableParams(result.Ref(), ctx, graphParams, secureParams);
- settings->EnableFullResultWrite = enableFullResultWrite;
+ bool enableFullResultWrite = settings->EnableFullResultWrite.Get().GetOrElse(false);
+ if (enableFullResultWrite) {
+ const auto type = result.Input().Ref().GetTypeAnn();
+ const auto integration = GetDqIntegrationForFullResTable(State);
+ enableFullResultWrite = type->GetKind() == ETypeAnnotationKind::List
+ && type->Cast<TListExprType>()->GetItemType()->GetKind() == ETypeAnnotationKind::Struct
+ && !fillSettings.Discard
+ && integration
+ && integration->PrepareFullResultTableParams(result.Ref(), ctx, graphParams, secureParams);
+ settings->EnableFullResultWrite = enableFullResultWrite;
}
// bool executeUdfLocallyIfPossible ?
@@ -700,8 +700,8 @@ private:
auto future = localRun
? TLocalExecutor(State).Execute(ctx.GetPosition(input->Pos()), lambda, columns, secureParams, fillSettings)
: State->DqGateway->ExecutePlan(
- State->SessionId, *executionPlanner.Get(), columns, secureParams, graphParams,
- settings, progressWriter, ModulesMapping, fillSettings.Discard);
+ State->SessionId, *executionPlanner.Get(), columns, secureParams, graphParams,
+ settings, progressWriter, ModulesMapping, fillSettings.Discard);
if (State->Metrics) {
State->Metrics->IncCounter("dq", localRun
@@ -711,13 +711,13 @@ private:
FlushStatisticsToState();
- return WrapFutureCallback(future, [localRun, startTime, type, fillSettings, level, settings, enableFullResultWrite, columns, graphParams, state = State](const IDqGateway::TResult& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- YQL_LOG(DEBUG) << state->SessionId << " WrapFutureCallback";
+ return WrapFutureCallback(future, [localRun, startTime, type, fillSettings, level, settings, enableFullResultWrite, columns, graphParams, state = State](const IDqGateway::TResult& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ YQL_LOG(DEBUG) << state->SessionId << " WrapFutureCallback";
auto duration = TInstant::Now() - startTime;
- if (state->Metrics) {
- state->Metrics->SetCounter("dq", "TotalExecutionTime", duration.MilliSeconds());
- state->Metrics->SetCounter(
+ if (state->Metrics) {
+ state->Metrics->SetCounter("dq", "TotalExecutionTime", duration.MilliSeconds());
+ state->Metrics->SetCounter(
"dq",
localRun
? "InMemoryExecutionTime"
@@ -725,10 +725,10 @@ private:
duration.MilliSeconds());
}
- state->Statistics[state->MetricId++] = res.Statistics;
+ state->Statistics[state->MetricId++] = res.Statistics;
if (res.Fallback) {
- if (state->Settings->FallbackPolicy.Get().GetOrElse("default") == "never" || state->TypeCtx->ForceDq) {
+ if (state->Settings->FallbackPolicy.Get().GetOrElse("default") == "never" || state->TypeCtx->ForceDq) {
auto issues = TIssues{TIssue(ctx.GetPosition(input->Pos()), "Gateway Error").SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_WARNING)};
issues.AddIssues(res.Issues);
ctx.AssociativeIssues.emplace(input.Get(), std::move(issues));
@@ -736,16 +736,16 @@ private:
}
YQL_LOG(DEBUG) << "Fallback from gateway: " << NCommon::ExprToPrettyString(ctx, *input);
- TIssue warning(ctx.GetPosition(input->Pos()), "DQ cannot execute the query");
+ TIssue warning(ctx.GetPosition(input->Pos()), "DQ cannot execute the query");
warning.Severity = TSeverityIds::S_INFO;
- ctx.IssueManager.RaiseIssue(warning);
+ ctx.IssueManager.RaiseIssue(warning);
if (res.ForceFallback) {
- state->Metrics->IncCounter("dq", "ForceFallback");
+ state->Metrics->IncCounter("dq", "ForceFallback");
}
- return FallbackCallback(state, input, output, ctx);
+ return FallbackCallback(state, input, output, ctx);
}
-
+
output = input;
input->SetState(TExprNode::EState::ExecutionComplete);
@@ -785,13 +785,13 @@ private:
}
}
writer.OnEndList();
- if (enableFullResultWrite) {
+ if (enableFullResultWrite) {
writer.OnKeyedItem("Ref");
writer.OnBeginList();
writer.OnListItem();
- const auto integration = GetDqIntegrationForFullResTable(state);
- YQL_ENSURE(integration);
- integration->WriteFullResultTableRef(writer, columns, graphParams);
+ const auto integration = GetDqIntegrationForFullResTable(state);
+ YQL_ENSURE(integration);
+ integration->WriteFullResultTableRef(writer, columns, graphParams);
writer.OnEndList();
}
writer.OnKeyedItem("Truncated");
@@ -816,7 +816,7 @@ private:
}, "");
} catch (...) {
- ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), CurrentExceptionMessage()));
+ ctx.AddError(TIssue(ctx.GetPosition(input->Pos()), CurrentExceptionMessage()));
return SyncError();
}
}
@@ -832,11 +832,11 @@ private:
}
TStatusCallbackPair HandlePull(const TExprNode::TPtr& input, TExprContext& ctx) {
- YQL_CLOG(DEBUG, ProviderDq) << "Executing " << input->Content() << " (UniqueId=" << input->UniqueId() << ")";
- YQL_CLOG(TRACE, ProviderDq) << "HandlePull " << NCommon::ExprToPrettyString(ctx, *input);
+ YQL_CLOG(DEBUG, ProviderDq) << "Executing " << input->Content() << " (UniqueId=" << input->UniqueId() << ")";
+ YQL_CLOG(TRACE, ProviderDq) << "HandlePull " << NCommon::ExprToPrettyString(ctx, *input);
TInstant startTime = TInstant::Now();
- auto pull = TPull(input);
+ auto pull = TPull(input);
YQL_ENSURE(!TMaybeNode<TDqQuery>(pull.Input().Ptr()) || State->Settings->EnableComputeActor.Get().GetOrElse(false),
"DqQuery is not supported with worker actor");
@@ -891,20 +891,20 @@ private:
TDqsPipelineConfigurator peepholeConfig;
TPeepholeSettings peepholeSettings;
peepholeSettings.CommonConfig = &peepholeConfig;
- bool hasNonDeterministicFunctions;
- // TODO: do it per stage
+ bool hasNonDeterministicFunctions;
+ // TODO: do it per stage
auto status = PeepHoleOptimizeNode<true>(optimizedInput, optimizedInput, ctx, *State->TypeCtx, nullptr, hasNonDeterministicFunctions, peepholeSettings);
- if (status != TStatus::Ok) {
- ctx.AddError(TIssue(ctx.GetPosition(optimizedInput->Pos()), TString("Peephole optimization failed for Dq stage")));
- return SyncStatus(status);
+ if (status != TStatus::Ok) {
+ ctx.AddError(TIssue(ctx.GetPosition(optimizedInput->Pos()), TString("Peephole optimization failed for Dq stage")));
+ return SyncStatus(status);
}
- YQL_CLOG(TRACE, ProviderDq) << "After PeepHole\n" << NCommon::ExprToPrettyString(ctx, *optimizedInput);
+ YQL_CLOG(TRACE, ProviderDq) << "After PeepHole\n" << NCommon::ExprToPrettyString(ctx, *optimizedInput);
// copy-paste {
TUserDataTable crutches = State->TypeCtx->UserDataStorageCrutches;
TUserDataTable files;
StartCounter("FreezeUsedFiles");
- auto filesRes = NCommon::FreezeUsedFiles(*optimizedInput, files, *State->TypeCtx, ctx, [](const TString&){return true;}, crutches);
+ auto filesRes = NCommon::FreezeUsedFiles(*optimizedInput, files, *State->TypeCtx, ctx, [](const TString&){return true;}, crutches);
if (filesRes.first.Level != TStatus::Ok) {
if (filesRes.first.Level != TStatus::Error) {
YQL_LOG(DEBUG) << "Freezing files for " << input->Content() << " (UniqueId=" << input->UniqueId() << ")";
@@ -916,7 +916,7 @@ private:
auto executionPlanner = MakeHolder<TDqsExecutionPlanner>(
State->TypeCtx, ctx, State->FunctionRegistry,
- optimizedInput);
+ optimizedInput);
// exprRoot must be DqCnResult or DqQuery
@@ -946,15 +946,15 @@ private:
YQL_ENSURE(stagesCount <= maxTasksPerOperation);
- try {
- while (executionPlanner->PlanExecution(settings, canFallback) > maxTasksPerOperation && tasksPerStage > 1) {
- tasksPerStage /= 2;
- settings->MaxTasksPerStage = tasksPerStage;
- executionPlanner->Clear();
- }
- } catch (const TFallbackError& err) {
- YQL_ENSURE(canFallback, "Unexpected TFallbackError: " << err.what());
- return FallbackWithMessage(pull.Ref(), err.what(), ctx);
+ try {
+ while (executionPlanner->PlanExecution(settings, canFallback) > maxTasksPerOperation && tasksPerStage > 1) {
+ tasksPerStage /= 2;
+ settings->MaxTasksPerStage = tasksPerStage;
+ executionPlanner->Clear();
+ }
+ } catch (const TFallbackError& err) {
+ YQL_ENSURE(canFallback, "Unexpected TFallbackError: " << err.what());
+ return FallbackWithMessage(pull.Ref(), err.what(), ctx);
}
bool fallbackFlag = false;
@@ -1000,7 +1000,7 @@ private:
}
}
- MarkProgressStarted(allPublicIds, State->ProgressWriter);
+ MarkProgressStarted(allPublicIds, State->ProgressWriter);
if (fallbackFlag) {
return FallbackWithMessage(pull.Ref(), "Too big attachment", ctx);
@@ -1013,53 +1013,53 @@ private:
settings->OptLLVM = *optLLVM;
}
- auto graphParams = GatherGraphParams(optimizedInput);
-
- bool ref = NCommon::HasResOrPullOption(pull.Ref(), "ref");
- bool autoRef = NCommon::HasResOrPullOption(pull.Ref(), "autoref");
-
- bool enableFullResultWrite = settings->EnableFullResultWrite.Get().GetOrElse(false);
- if (enableFullResultWrite) {
- const auto integration = GetDqIntegrationForFullResTable(State);
- enableFullResultWrite = (ref || autoRef)
- && !fillSettings.Discard
- && integration
- && integration->PrepareFullResultTableParams(pull.Ref(), ctx, graphParams, secureParams);
- settings->EnableFullResultWrite = enableFullResultWrite;
- }
-
- if (ref) {
- if (!enableFullResultWrite) {
- return FallbackWithMessage(pull.Ref(),
- TStringBuilder() << "RefSelect mode cannot be used with DQ, because \"" << State->TypeCtx->FullResultDataSink << "\" provider has failed to prepare a result table",
- ctx);
- }
- // Force write to table
- settings->_AllResultsBytesLimit = 0;
- settings->_RowsLimitPerWrite = 0;
+ auto graphParams = GatherGraphParams(optimizedInput);
+
+ bool ref = NCommon::HasResOrPullOption(pull.Ref(), "ref");
+ bool autoRef = NCommon::HasResOrPullOption(pull.Ref(), "autoref");
+
+ bool enableFullResultWrite = settings->EnableFullResultWrite.Get().GetOrElse(false);
+ if (enableFullResultWrite) {
+ const auto integration = GetDqIntegrationForFullResTable(State);
+ enableFullResultWrite = (ref || autoRef)
+ && !fillSettings.Discard
+ && integration
+ && integration->PrepareFullResultTableParams(pull.Ref(), ctx, graphParams, secureParams);
+ settings->EnableFullResultWrite = enableFullResultWrite;
}
+ if (ref) {
+ if (!enableFullResultWrite) {
+ return FallbackWithMessage(pull.Ref(),
+ TStringBuilder() << "RefSelect mode cannot be used with DQ, because \"" << State->TypeCtx->FullResultDataSink << "\" provider has failed to prepare a result table",
+ ctx);
+ }
+ // Force write to table
+ settings->_AllResultsBytesLimit = 0;
+ settings->_RowsLimitPerWrite = 0;
+ }
+
IDqGateway::TDqProgressWriter progressWriter = MakeDqProgressWriter(allPublicIds);
- auto future = State->DqGateway->ExecutePlan(State->SessionId, *executionPlanner.Get(), columns, secureParams, graphParams,
- settings, progressWriter, ModulesMapping, fillSettings.Discard);
+ auto future = State->DqGateway->ExecutePlan(State->SessionId, *executionPlanner.Get(), columns, secureParams, graphParams,
+ settings, progressWriter, ModulesMapping, fillSettings.Discard);
- future.Subscribe([allPublicIds, progressWriter = State->ProgressWriter](const NThreading::TFuture<IDqGateway::TResult>& completedFuture) {
+ future.Subscribe([allPublicIds, progressWriter = State->ProgressWriter](const NThreading::TFuture<IDqGateway::TResult>& completedFuture) {
YQL_ENSURE(!completedFuture.HasException());
- MarkProgressFinished(allPublicIds, completedFuture.GetValueSync().Success(), progressWriter);
+ MarkProgressFinished(allPublicIds, completedFuture.GetValueSync().Success(), progressWriter);
});
- executionPlanner.Destroy();
+ executionPlanner.Destroy();
int level = 0;
// TODO: remove copy-paste
- return WrapFutureCallback(future, [settings, startTime, localRun, type, fillSettings, level, graphParams, columns, enableFullResultWrite, state = State](const IDqGateway::TResult& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
- YQL_LOG(DEBUG) << state->SessionId << " WrapFutureCallback";
+ return WrapFutureCallback(future, [settings, startTime, localRun, type, fillSettings, level, graphParams, columns, enableFullResultWrite, state = State](const IDqGateway::TResult& res, const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
+ YQL_LOG(DEBUG) << state->SessionId << " WrapFutureCallback";
auto duration = TInstant::Now() - startTime;
- if (state->Metrics) {
- state->Metrics->SetCounter("dq", "TotalExecutionTime", duration.MilliSeconds());
- state->Metrics->SetCounter(
+ if (state->Metrics) {
+ state->Metrics->SetCounter("dq", "TotalExecutionTime", duration.MilliSeconds());
+ state->Metrics->SetCounter(
"dq",
localRun
? "InMemoryExecutionTime"
@@ -1070,10 +1070,10 @@ private:
state->Statistics[state->MetricId++] = res.Statistics;
if (res.Fallback) {
- if (state->Metrics) {
- state->Metrics->IncCounter("dq", "Fallback");
+ if (state->Metrics) {
+ state->Metrics->IncCounter("dq", "Fallback");
}
- state->Statistics[state->MetricId++].Entries.push_back(TOperationStatistics::TEntry("Fallback", 0, 0, 0, 0, 1));
+ state->Statistics[state->MetricId++].Entries.push_back(TOperationStatistics::TEntry("Fallback", 0, 0, 0, 0, 1));
// never fallback will be captured in yql_facade
auto issues = TIssues{TIssue(ctx.GetPosition(input->Pos()), "Gateway Error").SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_WARNING)};
issues.AddIssues(res.Issues);
@@ -1103,8 +1103,8 @@ private:
const bool truncated = res.Truncated;
const ui64 rowsCount = res.RowsCount;
- if (truncated && !state->TypeCtx->ForceDq && !enableFullResultWrite) {
- auto issue = TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "DQ cannot execute the query. Cause: " << "too big result " << trStr).SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_INFO);
+ if (truncated && !state->TypeCtx->ForceDq && !enableFullResultWrite) {
+ auto issue = TIssue(ctx.GetPosition(input->Pos()), TStringBuilder() << "DQ cannot execute the query. Cause: " << "too big result " << trStr).SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_INFO);
bool error = settings->FallbackPolicy.Get().GetOrElse("default") == "never";
for (const auto& i : res.Issues) {
TIssuePtr subIssue = new TIssue(i);
@@ -1118,7 +1118,7 @@ private:
issue.Message = "Too big result " + trStr;
issue.Severity = TSeverityIds::S_ERROR;
}
- ctx.IssueManager.RaiseIssue(issue);
+ ctx.IssueManager.RaiseIssue(issue);
return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Error);
}
@@ -1142,13 +1142,13 @@ private:
}
writer.OnEndList();
- if (enableFullResultWrite) {
+ if (enableFullResultWrite) {
writer.OnKeyedItem("Ref");
writer.OnBeginList();
writer.OnListItem();
- const auto integration = GetDqIntegrationForFullResTable(state);
- YQL_ENSURE(integration);
- integration->WriteFullResultTableRef(writer, columns, graphParams);
+ const auto integration = GetDqIntegrationForFullResTable(state);
+ YQL_ENSURE(integration);
+ integration->WriteFullResultTableRef(writer, columns, graphParams);
writer.OnEndList();
}
@@ -1165,21 +1165,21 @@ private:
writer.OnEndMap();
- ctx.IssueManager.RaiseIssues(res.Issues);
+ ctx.IssueManager.RaiseIssues(res.Issues);
input->SetResult(ctx.NewAtom(input->Pos(), out.Str()));
return IGraphTransformer::TStatus(IGraphTransformer::TStatus::Ok);
}, "");
}
IDqGateway::TDqProgressWriter MakeDqProgressWriter(const THashMap<ui32, ui32>& allPublicIds) const {
- IDqGateway::TDqProgressWriter dqProgressWriter = [progressWriter = State->ProgressWriter, allPublicIds](const TString& stage) {
+ IDqGateway::TDqProgressWriter dqProgressWriter = [progressWriter = State->ProgressWriter, allPublicIds](const TString& stage) {
for (const auto& publicId : allPublicIds) {
auto p = TOperationProgress(TString(DqProviderName), publicId.first, TOperationProgress::EState::InProgress, stage);
if (publicId.second) {
p.Counters.ConstructInPlace();
p.Counters->Running = p.Counters->Total = publicId.second;
}
- progressWriter(p);
+ progressWriter(p);
}
};
return dqProgressWriter;
@@ -1192,7 +1192,7 @@ private:
p.Counters.ConstructInPlace();
p.Counters->Running = p.Counters->Total = publicId.second;
}
- progressWriter(p);
+ progressWriter(p);
}
}
@@ -1204,7 +1204,7 @@ private:
p.Counters.ConstructInPlace();
(success ? p.Counters->Completed : p.Counters->Failed) = p.Counters->Total = publicId.second;
}
- progressWriter(p);
+ progressWriter(p);
}
}
@@ -1218,42 +1218,42 @@ private:
}
}
- THashMap<TString, TString> GatherGraphParams(const TExprNode::TPtr& root) {
- THashMap<TString, TString> params;
- VisitExpr(root, [&](const TExprNode::TPtr& node) -> bool {
- if (node->IsCallable()) {
- for (const auto& provider : State->TypeCtx->DataSources) {
- if (provider->CanParse(*node)) {
- if (auto dqIntegration = provider->GetDqIntegration()) {
- dqIntegration->Annotate(*node, params);
- return false;
- }
- }
- }
-
- for (const auto& provider : State->TypeCtx->DataSinks) {
- if (provider->CanParse(*node)) {
- if (auto dqIntegration = provider->GetDqIntegration()) {
- dqIntegration->Annotate(*node, params);
- return false;
- }
- }
- }
- }
- return true;
- });
- return params;
- }
-
- static IDqIntegration* GetDqIntegrationForFullResTable(const TDqStatePtr& state) {
- if (auto fullResultTableProvider = state->TypeCtx->DataSinkMap.Value(state->TypeCtx->FullResultDataSink, nullptr)) {
- auto dqIntegration = fullResultTableProvider->GetDqIntegration();
- YQL_ENSURE(dqIntegration);
- return dqIntegration;
- }
- return nullptr;
- }
-
+ THashMap<TString, TString> GatherGraphParams(const TExprNode::TPtr& root) {
+ THashMap<TString, TString> params;
+ VisitExpr(root, [&](const TExprNode::TPtr& node) -> bool {
+ if (node->IsCallable()) {
+ for (const auto& provider : State->TypeCtx->DataSources) {
+ if (provider->CanParse(*node)) {
+ if (auto dqIntegration = provider->GetDqIntegration()) {
+ dqIntegration->Annotate(*node, params);
+ return false;
+ }
+ }
+ }
+
+ for (const auto& provider : State->TypeCtx->DataSinks) {
+ if (provider->CanParse(*node)) {
+ if (auto dqIntegration = provider->GetDqIntegration()) {
+ dqIntegration->Annotate(*node, params);
+ return false;
+ }
+ }
+ }
+ }
+ return true;
+ });
+ return params;
+ }
+
+ static IDqIntegration* GetDqIntegrationForFullResTable(const TDqStatePtr& state) {
+ if (auto fullResultTableProvider = state->TypeCtx->DataSinkMap.Value(state->TypeCtx->FullResultDataSink, nullptr)) {
+ auto dqIntegration = fullResultTableProvider->GetDqIntegration();
+ YQL_ENSURE(dqIntegration);
+ return dqIntegration;
+ }
+ return nullptr;
+ }
+
TDqStatePtr State;
THolder<IGraphTransformer> DqTypeAnnotationTransformer;
mutable THashMap<TString, TFileLinkPtr> FileLinks;
diff --git a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h
index e6fdd644a1..1560162dbc 100644
--- a/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h
+++ b/ydb/library/yql/providers/dq/provider/exec/yql_dq_exectransformer.h
@@ -2,11 +2,11 @@
#include <ydb/library/yql/core/yql_data_provider.h>
-#include <util/generic/ptr.h>
-
+#include <util/generic/ptr.h>
+
namespace NYql {
struct TDqState;
- using TDqStatePtr = TIntrusivePtr<TDqState>;
+ using TDqStatePtr = TIntrusivePtr<TDqState>;
IGraphTransformer* CreateInMemoryExecTransformer(const TDqStatePtr& state);
} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/provider/ya.make b/ydb/library/yql/providers/dq/provider/ya.make
index bdb3547266..6321f3096d 100644
--- a/ydb/library/yql/providers/dq/provider/ya.make
+++ b/ydb/library/yql/providers/dq/provider/ya.make
@@ -5,10 +5,10 @@ OWNER(g:yql)
SRCS(
yql_dq_control.cpp
yql_dq_control.h
- yql_dq_datasink_type_ann.cpp
- yql_dq_datasink_type_ann.h
- yql_dq_datasource_type_ann.cpp
- yql_dq_datasource_type_ann.h
+ yql_dq_datasink_type_ann.cpp
+ yql_dq_datasink_type_ann.h
+ yql_dq_datasource_type_ann.cpp
+ yql_dq_datasource_type_ann.h
yql_dq_gateway.cpp
yql_dq_gateway.h
yql_dq_provider.cpp
@@ -17,8 +17,8 @@ SRCS(
yql_dq_datasink.h
yql_dq_datasource.cpp
yql_dq_datasource.h
- yql_dq_recapture.cpp
- yql_dq_recapture.h
+ yql_dq_recapture.cpp
+ yql_dq_recapture.h
)
PEERDIR(
@@ -45,6 +45,6 @@ PEERDIR(
ydb/library/yql/providers/result/expr_nodes
)
-YQL_LAST_ABI_VERSION()
+YQL_LAST_ABI_VERSION()
END()
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_control.h b/ydb/library/yql/providers/dq/provider/yql_dq_control.h
index d6b58eacd7..3b1b7e05c7 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_control.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_control.h
@@ -7,10 +7,10 @@
namespace NYql {
-namespace NProto {
-class TDqConfig;
-}
-
+namespace NProto {
+class TDqConfig;
+}
+
class IDqControl : public TThrRefBase {
public:
using TPtr = TIntrusivePtr<IDqControl>;
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp
index a8b5e598a0..f67a86786d 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.cpp
@@ -1,7 +1,7 @@
#include "yql_dq_datasink.h"
#include "yql_dq_state.h"
-#include "yql_dq_datasink_type_ann.h"
-#include "yql_dq_recapture.h"
+#include "yql_dq_datasink_type_ann.h"
+#include "yql_dq_recapture.h"
#include <ydb/library/yql/providers/dq/opt/logical_optimize.h>
#include <ydb/library/yql/providers/dq/opt/physical_optimize.h>
@@ -21,17 +21,17 @@
namespace NYql {
-using namespace NNodes;
-
+using namespace NNodes;
+
class TDqDataProviderSink: public TDataProviderBase {
public:
TDqDataProviderSink(const TDqStatePtr& state)
: State(state)
, LogOptTransformer([state] () { return CreateDqsLogOptTransformer(/*TODO: State->TypeCtx);*/nullptr, state->Settings); })
- , PhyOptTransformer([] () { return CreateDqsPhyOptTransformer(/*TODO: State->TypeCtx*/nullptr); })
- , PhysicalFinalizingTransformer([] () { return CreateDqsFinalizingOptTransformer(); })
+ , PhyOptTransformer([] () { return CreateDqsPhyOptTransformer(/*TODO: State->TypeCtx*/nullptr); })
+ , PhysicalFinalizingTransformer([] () { return CreateDqsFinalizingOptTransformer(); })
, TypeAnnotationTransformer([state] () { return CreateDqsDataSinkTypeAnnotationTransformer(state->TypeCtx); })
- , RecaptureTransformer([state] () { return CreateDqsRecaptureTransformer(state); })
+ , RecaptureTransformer([state] () { return CreateDqsRecaptureTransformer(state); })
{ }
bool CollectStatistics(NYson::TYsonWriter& writer, bool totalOnly) override {
@@ -151,57 +151,57 @@ public:
return true;
}
- bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
- if (node.IsCallable(TCoDataSink::CallableName())) {
- if (!EnsureMinMaxArgsCount(node, 1, 2, ctx)) {
- return false;
- }
-
+ bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
+ if (node.IsCallable(TCoDataSink::CallableName())) {
+ if (!EnsureMinMaxArgsCount(node, 1, 2, ctx)) {
+ return false;
+ }
+
if (node.Child(0)->Content() == DqProviderName) {
- if (node.ChildrenSize() == 2) {
- if (!EnsureAtom(*node.Child(1), ctx)) {
- return false;
- }
-
- if (node.Child(1)->Content() != "$all") {
- ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << "Unexpected cluster name: " << node.Child(1)->Content()));
- return false;
- }
- }
- cluster = Nothing();
- return true;
- }
- }
-
- ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid DQ DataSource parameters"));
- return false;
- }
-
- bool CanParse(const TExprNode& node) override {
- return TypeAnnotationTransformer->CanParse(node);
- }
-
+ if (node.ChildrenSize() == 2) {
+ if (!EnsureAtom(*node.Child(1), ctx)) {
+ return false;
+ }
+
+ if (node.Child(1)->Content() != "$all") {
+ ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << "Unexpected cluster name: " << node.Child(1)->Content()));
+ return false;
+ }
+ }
+ cluster = Nothing();
+ return true;
+ }
+ }
+
+ ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid DQ DataSource parameters"));
+ return false;
+ }
+
+ bool CanParse(const TExprNode& node) override {
+ return TypeAnnotationTransformer->CanParse(node);
+ }
+
IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override {
Y_UNUSED(instantOnly);
return *TypeAnnotationTransformer;
}
- IGraphTransformer& GetRecaptureOptProposalTransformer() override {
- return *RecaptureTransformer;
- }
-
+ IGraphTransformer& GetRecaptureOptProposalTransformer() override {
+ return *RecaptureTransformer;
+ }
+
IGraphTransformer& GetLogicalOptProposalTransformer() override {
- return *LogOptTransformer;
+ return *LogOptTransformer;
}
IGraphTransformer& GetPhysicalOptProposalTransformer() override {
- return *PhyOptTransformer;
+ return *PhyOptTransformer;
}
-
+
IGraphTransformer& GetPhysicalFinalizingTransformer() override {
- return *PhysicalFinalizingTransformer;
+ return *PhysicalFinalizingTransformer;
}
-
+
TStringBuf GetName() const override {
return DqProviderName;
}
@@ -269,9 +269,9 @@ public:
TLazyInitHolder<IGraphTransformer> LogOptTransformer;
TLazyInitHolder<IGraphTransformer> PhyOptTransformer;
- TLazyInitHolder<IGraphTransformer> PhysicalFinalizingTransformer;
- TLazyInitHolder<TVisitorTransformerBase> TypeAnnotationTransformer;
- TLazyInitHolder<IGraphTransformer> RecaptureTransformer;
+ TLazyInitHolder<IGraphTransformer> PhysicalFinalizingTransformer;
+ TLazyInitHolder<TVisitorTransformerBase> TypeAnnotationTransformer;
+ TLazyInitHolder<IGraphTransformer> RecaptureTransformer;
};
TIntrusivePtr<IDataProvider> CreateDqDataSink(const TDqStatePtr& state) {
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.h b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.h
index d9849824a7..4094e3f3e1 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_datasink.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasink.h
@@ -4,7 +4,7 @@
namespace NYql {
struct TDqState;
- using TDqStatePtr = TIntrusivePtr<TDqState>;
+ using TDqStatePtr = TIntrusivePtr<TDqState>;
TIntrusivePtr<IDataProvider> CreateDqDataSink(const TDqStatePtr& state);
} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.cpp
index c8d9786a7a..66ad10577f 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.cpp
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.cpp
@@ -1,40 +1,40 @@
-#include "yql_dq_datasink_type_ann.h"
-
+#include "yql_dq_datasink_type_ann.h"
+
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
#include <ydb/library/yql/dq/type_ann/dq_type_ann.h>
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
-
-namespace NYql {
-
-using namespace NNodes;
-
-namespace {
-
-class TDqsDataSinkTypeAnnotationTransformer : public TVisitorTransformerBase {
-public:
+
+namespace NYql {
+
+using namespace NNodes;
+
+namespace {
+
+class TDqsDataSinkTypeAnnotationTransformer : public TVisitorTransformerBase {
+public:
TDqsDataSinkTypeAnnotationTransformer(TTypeAnnotationContext* typeCtx)
: TVisitorTransformerBase(true), TypeCtx(typeCtx)
- {
- AddHandler({TDqStage::CallableName()}, Hndl(&NDq::AnnotateDqStage));
- AddHandler({TDqPhyStage::CallableName()}, Hndl(&NDq::AnnotateDqPhyStage));
- AddHandler({TDqOutput::CallableName()}, Hndl(&NDq::AnnotateDqOutput));
- AddHandler({TDqCnUnionAll::CallableName()}, Hndl(&NDq::AnnotateDqConnection));
+ {
+ AddHandler({TDqStage::CallableName()}, Hndl(&NDq::AnnotateDqStage));
+ AddHandler({TDqPhyStage::CallableName()}, Hndl(&NDq::AnnotateDqPhyStage));
+ AddHandler({TDqOutput::CallableName()}, Hndl(&NDq::AnnotateDqOutput));
+ AddHandler({TDqCnUnionAll::CallableName()}, Hndl(&NDq::AnnotateDqConnection));
AddHandler({TDqCnHashShuffle::CallableName()}, Hndl(&NDq::AnnotateDqCnHashShuffle));
- AddHandler({TDqCnResult::CallableName()}, Hndl(&NDq::AnnotateDqCnResult));
- AddHandler({TDqCnMap::CallableName()}, Hndl(&NDq::AnnotateDqConnection));
- AddHandler({TDqCnBroadcast::CallableName()}, Hndl(&NDq::AnnotateDqConnection));
- AddHandler({TDqCnValue::CallableName()}, Hndl(&NDq::AnnotateDqCnValue));
+ AddHandler({TDqCnResult::CallableName()}, Hndl(&NDq::AnnotateDqCnResult));
+ AddHandler({TDqCnMap::CallableName()}, Hndl(&NDq::AnnotateDqConnection));
+ AddHandler({TDqCnBroadcast::CallableName()}, Hndl(&NDq::AnnotateDqConnection));
+ AddHandler({TDqCnValue::CallableName()}, Hndl(&NDq::AnnotateDqCnValue));
AddHandler({TDqCnMerge::CallableName()}, Hndl(&NDq::AnnotateDqCnMerge));
- AddHandler({TDqReplicate::CallableName()}, Hndl(&NDq::AnnotateDqReplicate));
- AddHandler({TDqJoin::CallableName()}, Hndl(&NDq::AnnotateDqJoin));
+ AddHandler({TDqReplicate::CallableName()}, Hndl(&NDq::AnnotateDqReplicate));
+ AddHandler({TDqJoin::CallableName()}, Hndl(&NDq::AnnotateDqJoin));
AddHandler({TDqPhyMapJoin::CallableName()}, Hndl(&NDq::AnnotateDqMapOrDictJoin));
- AddHandler({TDqPhyCrossJoin::CallableName()}, Hndl(&NDq::AnnotateDqCrossJoin));
+ AddHandler({TDqPhyCrossJoin::CallableName()}, Hndl(&NDq::AnnotateDqCrossJoin));
AddHandler({TDqPhyJoinDict::CallableName()}, Hndl(&NDq::AnnotateDqMapOrDictJoin));
AddHandler({TDqSink::CallableName()}, Hndl(&NDq::AnnotateDqSink));
AddHandler({TDqWrite::CallableName()}, Hndl(&TDqsDataSinkTypeAnnotationTransformer::AnnotateDqWrite));
AddHandler({TDqQuery::CallableName()}, Hndl(&NDq::AnnotateDqQuery));
- }
+ }
private:
TStatus AnnotateDqWrite(const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx) {
@@ -67,12 +67,12 @@ private:
}
TTypeAnnotationContext* TypeCtx;
-};
-
-} // unnamed
-
+};
+
+} // unnamed
+
THolder<TVisitorTransformerBase> CreateDqsDataSinkTypeAnnotationTransformer(TTypeAnnotationContext* typeCtx) {
return THolder(new TDqsDataSinkTypeAnnotationTransformer(typeCtx));
-}
-
-} // NYql
+}
+
+} // NYql
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.h b/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.h
index df86def2cd..b6404837da 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasink_type_ann.h
@@ -1,12 +1,12 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/providers/common/transform/yql_visit.h>
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
-
-#include <util/generic/ptr.h>
-
-namespace NYql {
-
+
+#include <util/generic/ptr.h>
+
+namespace NYql {
+
THolder<TVisitorTransformerBase> CreateDqsDataSinkTypeAnnotationTransformer(TTypeAnnotationContext* typeCtx);
-
-} // NYql
+
+} // NYql
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasource.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_datasource.cpp
index aea55435de..0ea413dab7 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_datasource.cpp
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasource.cpp
@@ -1,5 +1,5 @@
#include "yql_dq_datasource.h"
-#include "yql_dq_datasource_type_ann.h"
+#include "yql_dq_datasource_type_ann.h"
#include "yql_dq_state.h"
#include <ydb/library/yql/providers/common/config/yql_configuration_transformer.h>
@@ -42,7 +42,7 @@ public:
return MakeHolder<NCommon::TProviderConfigurationTransformer>(State->Settings, *State->TypeCtx, TString{DqProviderName});
})
, ExecTransformer([this, execTransformerFactory] () { return THolder<IGraphTransformer>(execTransformerFactory(State)); })
- , TypeAnnotationTransformer([] () { return CreateDqsDataSourceTypeAnnotationTransformer(); })
+ , TypeAnnotationTransformer([] () { return CreateDqsDataSourceTypeAnnotationTransformer(); })
{ }
TStringBuf GetName() const override {
@@ -58,20 +58,20 @@ public:
return *ConfigurationTransformer;
}
- TExprNode::TPtr OptimizePull(const TExprNode::TPtr& node, const TFillSettings& fillSettings, TExprContext& ctx,
+ TExprNode::TPtr OptimizePull(const TExprNode::TPtr& node, const TFillSettings& fillSettings, TExprContext& ctx,
IOptimizationContext& optCtx) override
{
Y_UNUSED(optCtx);
Y_UNUSED(fillSettings);
- if (TDqCnResult::Match(node.Get())) {
- return node;
- }
+ if (TDqCnResult::Match(node.Get())) {
+ return node;
+ }
- if (!TDqCnUnionAll::Match(node.Get())) {
- ctx.AddError(TIssue(node->Pos(ctx), "Last connection must be union all"));
- return {};
- }
+ if (!TDqCnUnionAll::Match(node.Get())) {
+ ctx.AddError(TIssue(node->Pos(ctx), "Last connection must be union all"));
+ return {};
+ }
TExprNode::TListType worlds;
VisitExpr(node, [&worlds] (const TExprNode::TPtr& item) {
@@ -87,43 +87,43 @@ public:
for (const auto& w : worlds)
replaces.emplace(w.Get(), newWorld);
- return Build<TDqCnResult>(ctx, node->Pos())
- .Output()
- .Stage<TDqStage>()
- .Inputs()
+ return Build<TDqCnResult>(ctx, node->Pos())
+ .Output()
+ .Stage<TDqStage>()
+ .Inputs()
.Add(ctx.ReplaceNodes(TExprNode::TPtr(node), std::move(replaces)))
- .Build()
- .Program()
- .Args({"row"})
- .Body("row")
- .Build()
+ .Build()
+ .Program()
+ .Args({"row"})
+ .Body("row")
+ .Build()
.Settings(TDqStageSettings().BuildNode(ctx, node->Pos()))
- .Build()
- .Index().Build("0")
- .Build()
- .ColumnHints() // TODO: set column hints
- .Build()
- .Done().Ptr();
+ .Build()
+ .Index().Build("0")
+ .Build()
+ .ColumnHints() // TODO: set column hints
+ .Build()
+ .Done().Ptr();
}
bool CanPullResult(const TExprNode& node, TSyncMap& syncList, bool& canRef) override {
- if (!TDqCnUnionAll::Match(&node)) {
- return false;
- }
-
- if (auto type = GetItemType(*node.GetTypeAnn()); !type || type->GetKind() != ETypeAnnotationKind::Struct) {
+ if (!TDqCnUnionAll::Match(&node)) {
return false;
- }
-
- canRef = State->Settings->EnableFullResultWrite.Get().GetOrElse(false);
- if (canRef) {
- if (auto fullResultTableProvider = State->TypeCtx->DataSinkMap.Value(State->TypeCtx->FullResultDataSink, nullptr)) {
- canRef = !!fullResultTableProvider->GetDqIntegration();
- } else {
- canRef = false;
- }
- }
-
+ }
+
+ if (auto type = GetItemType(*node.GetTypeAnn()); !type || type->GetKind() != ETypeAnnotationKind::Struct) {
+ return false;
+ }
+
+ canRef = State->Settings->EnableFullResultWrite.Get().GetOrElse(false);
+ if (canRef) {
+ if (auto fullResultTableProvider = State->TypeCtx->DataSinkMap.Value(State->TypeCtx->FullResultDataSink, nullptr)) {
+ canRef = !!fullResultTableProvider->GetDqIntegration();
+ } else {
+ canRef = false;
+ }
+ }
+
for (const auto& child : node.ChildrenList())
VisitExpr(child, [&syncList] (const TExprNode::TPtr& item) {
if (ETypeAnnotationKind::World == item->GetTypeAnn()->GetKind()) {
@@ -135,30 +135,30 @@ public:
return true;
}
- bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
- if (node.IsCallable(TCoDataSource::CallableName())) {
- if (!EnsureMinMaxArgsCount(node, 1, 2, ctx)) {
- return false;
+ bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
+ if (node.IsCallable(TCoDataSource::CallableName())) {
+ if (!EnsureMinMaxArgsCount(node, 1, 2, ctx)) {
+ return false;
}
if (node.Child(0)->Content() == DqProviderName) {
- if (node.ChildrenSize() == 2) {
- if (!EnsureAtom(*node.Child(1), ctx)) {
- return false;
- }
-
- if (node.Child(1)->Content() != "$all") {
- ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << "Unexpected cluster name: " << node.Child(1)->Content()));
- return false;
- }
- }
- cluster = Nothing();
+ if (node.ChildrenSize() == 2) {
+ if (!EnsureAtom(*node.Child(1), ctx)) {
+ return false;
+ }
+
+ if (node.Child(1)->Content() != "$all") {
+ ctx.AddError(TIssue(ctx.GetPosition(node.Child(1)->Pos()), TStringBuilder() << "Unexpected cluster name: " << node.Child(1)->Content()));
+ return false;
+ }
+ }
+ cluster = Nothing();
return true;
}
}
- ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid DQ DataSource parameters"));
- return false;
+ ctx.AddError(TIssue(ctx.GetPosition(node.Pos()), "Invalid DQ DataSource parameters"));
+ return false;
}
bool CanExecute(const TExprNode& node) override {
@@ -166,7 +166,7 @@ public:
}
bool CanParse(const TExprNode& node) override {
- return TypeAnnotationTransformer->CanParse(node);
+ return TypeAnnotationTransformer->CanParse(node);
}
IGraphTransformer& GetCallableExecutionTransformer() override {
@@ -200,7 +200,7 @@ private:
TDqStatePtr State;
TLazyInitHolder<IGraphTransformer> ConfigurationTransformer;
TLazyInitHolder<IGraphTransformer> ExecTransformer;
- TLazyInitHolder<TVisitorTransformerBase> TypeAnnotationTransformer;
+ TLazyInitHolder<TVisitorTransformerBase> TypeAnnotationTransformer;
};
TIntrusivePtr<IDataProvider> CreateDqDataSource(const TDqStatePtr& state, TExecTransformerFactory execTransformerFactory) {
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasource.h b/ydb/library/yql/providers/dq/provider/yql_dq_datasource.h
index 500ec35d4a..cc933595f1 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_datasource.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasource.h
@@ -2,11 +2,11 @@
#include <ydb/library/yql/core/yql_data_provider.h>
-#include <util/generic/ptr.h>
-
+#include <util/generic/ptr.h>
+
namespace NYql {
struct TDqState;
- using TDqStatePtr = TIntrusivePtr<TDqState>;
+ using TDqStatePtr = TIntrusivePtr<TDqState>;
using TExecTransformerFactory = std::function<IGraphTransformer*(const TDqStatePtr& state)>;
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasource_type_ann.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_datasource_type_ann.cpp
index 6e03017ccb..13a697edd6 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_datasource_type_ann.cpp
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasource_type_ann.cpp
@@ -1,63 +1,63 @@
-#include "yql_dq_datasource_type_ann.h"
-
+#include "yql_dq_datasource_type_ann.h"
+
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
#include <ydb/library/yql/core/yql_expr_type_annotation.h>
#include <ydb/library/yql/dq/expr_nodes/dq_expr_nodes.h>
#include <ydb/library/yql/dq/type_ann/dq_type_ann.h>
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
-
-namespace NYql {
-
-using namespace NNodes;
-
-namespace {
-
-class TDqsDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase {
-public:
- TDqsDataSourceTypeAnnotationTransformer()
- : TVisitorTransformerBase(true)
- {
+
+namespace NYql {
+
+using namespace NNodes;
+
+namespace {
+
+class TDqsDataSourceTypeAnnotationTransformer : public TVisitorTransformerBase {
+public:
+ TDqsDataSourceTypeAnnotationTransformer()
+ : TVisitorTransformerBase(true)
+ {
AddHandler({TDqSourceWrap::CallableName()}, Hndl(&TDqsDataSourceTypeAnnotationTransformer::HandleSourceWrap<false>));
AddHandler({TDqSourceWideWrap::CallableName()}, Hndl(&TDqsDataSourceTypeAnnotationTransformer::HandleSourceWrap<true>));
AddHandler({TDqReadWrap::CallableName()}, Hndl(&TDqsDataSourceTypeAnnotationTransformer::HandleReadWrap));
AddHandler({TDqReadWideWrap::CallableName()}, Hndl(&TDqsDataSourceTypeAnnotationTransformer::HandleWideReadWrap));
- AddHandler({TCoConfigure::CallableName()}, Hndl(&TDqsDataSourceTypeAnnotationTransformer::HandleConfig));
+ AddHandler({TCoConfigure::CallableName()}, Hndl(&TDqsDataSourceTypeAnnotationTransformer::HandleConfig));
AddHandler({TDqSource::CallableName()}, Hndl(&NDq::AnnotateDqSource));
- }
-
-private:
- TStatus HandleReadWrap(const TExprNode::TPtr& input, TExprContext& ctx) {
+ }
+
+private:
+ TStatus HandleReadWrap(const TExprNode::TPtr& input, TExprContext& ctx) {
if (!EnsureMinMaxArgsCount(*input, 1, 3, ctx)) {
- return TStatus::Error;
- }
-
+ return TStatus::Error;
+ }
+
if (input->ChildrenSize() > TDqReadWrapBase::idx_Token) {
- if (!EnsureCallable(*input->Child(TDqReadWrapBase::idx_Token), ctx)) {
- return TStatus::Error;
- }
-
- if (!TCoSecureParam::Match(input->Child(TDqReadWrapBase::idx_Token))) {
- ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqReadWrapBase::idx_Token)->Pos()), TStringBuilder() << "Expected " << TCoSecureParam::CallableName()));
- return TStatus::Error;
- }
- }
-
+ if (!EnsureCallable(*input->Child(TDqReadWrapBase::idx_Token), ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!TCoSecureParam::Match(input->Child(TDqReadWrapBase::idx_Token))) {
+ ctx.AddError(TIssue(ctx.GetPosition(input->Child(TDqReadWrapBase::idx_Token)->Pos()), TStringBuilder() << "Expected " << TCoSecureParam::CallableName()));
+ return TStatus::Error;
+ }
+ }
+
if (input->ChildrenSize() > TDqReadWrapBase::idx_Flags && !EnsureTupleOfAtoms(*input->Child(TDqReadWrapBase::idx_Flags), ctx)) {
return TStatus::Error;
}
- if (!EnsureTupleTypeSize(input->Head(), 2, ctx)) {
- return TStatus::Error;
- }
+ if (!EnsureTupleTypeSize(input->Head(), 2, ctx)) {
+ return TStatus::Error;
+ }
const auto readerType = input->Head().GetTypeAnn()->Cast<TTupleExprType>()->GetItems().back();
- if (!EnsureListType(input->Head().Pos(), *readerType, ctx)) {
- return TStatus::Error;
- }
+ if (!EnsureListType(input->Head().Pos(), *readerType, ctx)) {
+ return TStatus::Error;
+ }
input->SetTypeAnn(readerType);
- return TStatus::Ok;
- }
-
+ return TStatus::Ok;
+ }
+
template<bool Wide>
TStatus HandleSourceWrap(const TExprNode::TPtr& input, TExprContext& ctx) {
if (!EnsureMinMaxArgsCount(*input, 3U, 4U, ctx)) {
@@ -137,29 +137,29 @@ private:
return TStatus::Ok;
}
- TStatus HandleConfig(const TExprNode::TPtr& input, TExprContext& ctx) {
- if (!EnsureMinArgsCount(*input, 2, ctx)) {
- return TStatus::Error;
- }
-
- if (!EnsureWorldType(*input->Child(TCoConfigure::idx_World), ctx)) {
- return TStatus::Error;
- }
-
+ TStatus HandleConfig(const TExprNode::TPtr& input, TExprContext& ctx) {
+ if (!EnsureMinArgsCount(*input, 2, ctx)) {
+ return TStatus::Error;
+ }
+
+ if (!EnsureWorldType(*input->Child(TCoConfigure::idx_World), ctx)) {
+ return TStatus::Error;
+ }
+
if (!EnsureSpecificDataSource(*input->Child(TCoConfigure::idx_DataSource), DqProviderName, ctx)) {
- return TStatus::Error;
- }
-
- input->SetTypeAnn(input->Child(TCoConfigure::idx_World)->GetTypeAnn());
-
- return TStatus::Ok;
- }
-};
-
-} // unnamed
-
-THolder<TVisitorTransformerBase> CreateDqsDataSourceTypeAnnotationTransformer() {
- return THolder(new TDqsDataSourceTypeAnnotationTransformer());
-}
-
-} // NYql
+ return TStatus::Error;
+ }
+
+ input->SetTypeAnn(input->Child(TCoConfigure::idx_World)->GetTypeAnn());
+
+ return TStatus::Ok;
+ }
+};
+
+} // unnamed
+
+THolder<TVisitorTransformerBase> CreateDqsDataSourceTypeAnnotationTransformer() {
+ return THolder(new TDqsDataSourceTypeAnnotationTransformer());
+}
+
+} // NYql
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_datasource_type_ann.h b/ydb/library/yql/providers/dq/provider/yql_dq_datasource_type_ann.h
index d31811f1bc..6638135f6b 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_datasource_type_ann.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_datasource_type_ann.h
@@ -1,14 +1,14 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/providers/common/transform/yql_visit.h>
-
-#include <util/generic/ptr.h>
-#include <util/generic/vector.h>
-
-namespace NYql {
-
-class TTypeAnnotationNode;
-
-THolder<TVisitorTransformerBase> CreateDqsDataSourceTypeAnnotationTransformer();
-
-} // NYql
+
+#include <util/generic/ptr.h>
+#include <util/generic/vector.h>
+
+namespace NYql {
+
+class TTypeAnnotationNode;
+
+THolder<TVisitorTransformerBase> CreateDqsDataSourceTypeAnnotationTransformer();
+
+} // NYql
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp
index 2341b28ad4..19bbab42f5 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.cpp
@@ -53,7 +53,7 @@ public:
void OnResponse(NThreading::TPromise<TResult> promise, TString sessionId, NGrpc::TGrpcStatus&& status, RespType&& resp, const THashMap<TString, TString>& modulesMapping, bool alwaysFallback = false)
{
YQL_LOG_CTX_SCOPE(sessionId);
- YQL_CLOG(TRACE, ProviderDq) << "TDqGateway::callback";
+ YQL_CLOG(TRACE, ProviderDq) << "TDqGateway::callback";
{
TGuard<TMutex> lock(ProgressMutex);
@@ -66,14 +66,14 @@ public:
bool fallback = false;
if (status.Ok()) {
- YQL_CLOG(TRACE, ProviderDq) << "TDqGateway::Ok";
+ YQL_CLOG(TRACE, ProviderDq) << "TDqGateway::Ok";
result.Truncated = resp.GetTruncated();
TOperationStatistics statistics;
for (const auto& t : resp.GetMetric()) {
- YQL_CLOG(TRACE, ProviderDq) << "Counter: " << t.GetName() << " : " << t.GetSum() << " : " << t.GetCount();
+ YQL_CLOG(TRACE, ProviderDq) << "Counter: " << t.GetName() << " : " << t.GetSum() << " : " << t.GetCount();
TOperationStatistics::TEntry entry(
t.GetName(),
t.GetSum(),
@@ -112,7 +112,7 @@ public:
result.Issues.AddIssues(issues);
result.SetSuccess();
} else {
- YQL_CLOG(ERROR, ProviderDq) << "Issue " << issues.ToString();
+ YQL_CLOG(ERROR, ProviderDq) << "Issue " << issues.ToString();
result.Issues.AddIssues(issues);
if (fallback) {
result.Fallback = true;
@@ -120,7 +120,7 @@ public:
}
}
} else {
- YQL_CLOG(ERROR, ProviderDq) << "Issue " << status.Msg;
+ YQL_CLOG(ERROR, ProviderDq) << "Issue " << status.Msg;
auto issue = TIssue(TStringBuilder{} << "Error " << status.GRpcStatusCode << " message: " << status.Msg);
result.Retriable = status.GRpcStatusCode == grpc::CANCELLED;
if ((status.GRpcStatusCode == grpc::UNAVAILABLE /* terminating state */
@@ -128,7 +128,7 @@ public:
|| status.GRpcStatusCode == grpc::RESOURCE_EXHAUSTED /* send message limit */
)
{
- YQL_CLOG(ERROR, ProviderDq) << "Fallback " << status.GRpcStatusCode;
+ YQL_CLOG(ERROR, ProviderDq) << "Fallback " << status.GRpcStatusCode;
result.Fallback = true;
result.SetSuccess();
result.Issues.AddIssue(issue.SetCode(TIssuesIds::DQ_GATEWAY_NEED_FALLBACK_ERROR, TSeverityIds::S_ERROR));
@@ -139,7 +139,7 @@ public:
}
if (error && alwaysFallback) {
- YQL_CLOG(ERROR, ProviderDq) << "Force Fallback";
+ YQL_CLOG(ERROR, ProviderDq) << "Force Fallback";
result.Fallback = true;
result.ForceFallback = true;
result.SetSuccess();
@@ -229,13 +229,13 @@ public:
}
NThreading::TFuture<TResult>
- ExecutePlan(const TString& sessionId, NDqs::IDqsExecutionPlanner& plan, const TVector<TString>& columns,
- const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& graphParams,
- const TDqSettings::TPtr& settings,
- const TDqProgressWriter& progressWriter, const THashMap<TString, TString>& modulesMapping,
- bool discard) override
+ ExecutePlan(const TString& sessionId, NDqs::IDqsExecutionPlanner& plan, const TVector<TString>& columns,
+ const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& graphParams,
+ const TDqSettings::TPtr& settings,
+ const TDqProgressWriter& progressWriter, const THashMap<TString, TString>& modulesMapping,
+ bool discard) override
{
- YQL_LOG_CTX_SCOPE(sessionId);
+ YQL_LOG_CTX_SCOPE(sessionId);
auto tasks = plan.GetTasks();
Yql::DqsProto::ExecuteGraphRequest queryPB;
@@ -259,19 +259,19 @@ public:
settings->Save(queryPB);
{
- auto& secParams = *queryPB.MutableSecureParams();
+ auto& secParams = *queryPB.MutableSecureParams();
for (const auto&[k, v] : secureParams) {
secParams[k] = v;
}
}
- {
- auto& gParams = *queryPB.MutableGraphParams();
- for (const auto&[k, v] : graphParams) {
- gParams[k] = v;
- }
- }
-
+ {
+ auto& gParams = *queryPB.MutableGraphParams();
+ for (const auto&[k, v] : graphParams) {
+ gParams[k] = v;
+ }
+ }
+
queryPB.SetDiscard(discard);
int retry = settings->MaxRetries.Get().GetOrElse(5);
@@ -293,8 +293,8 @@ public:
}
NThreading::TFuture<void> OpenSession(const TString& sessionId, const TString& username) override {
- YQL_LOG_CTX_SCOPE(sessionId);
- YQL_CLOG(INFO, ProviderDq) << "OpenSession";
+ YQL_LOG_CTX_SCOPE(sessionId);
+ YQL_CLOG(INFO, ProviderDq) << "OpenSession";
Yql::DqsProto::OpenSessionRequest request;
request.SetSession(sessionId);
request.SetUsername(username);
@@ -305,13 +305,13 @@ public:
auto promise = NThreading::NewPromise<void>();
auto callback = [this, promise, sessionId](NGrpc::TGrpcStatus&& status, Yql::DqsProto::OpenSessionResponse&& resp) mutable {
Y_UNUSED(resp);
- YQL_LOG_CTX_SCOPE(sessionId);
+ YQL_LOG_CTX_SCOPE(sessionId);
if (status.Ok()) {
- YQL_CLOG(INFO, ProviderDq) << "OpenSession OK";
+ YQL_CLOG(INFO, ProviderDq) << "OpenSession OK";
SchedulePingSessionRequest(sessionId);
Async([promise=std::move(promise)]() mutable { promise.SetValue(); });
} else {
- YQL_CLOG(ERROR, ProviderDq) << "OpenSession error: " << status.Msg;
+ YQL_CLOG(ERROR, ProviderDq) << "OpenSession error: " << status.Msg;
Async([promise=std::move(promise), status]() mutable { promise.SetException(status.Msg); });
}
};
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h
index d562776033..1185e91f8d 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_gateway.h
@@ -18,10 +18,10 @@
namespace NYql {
-namespace NProto {
-class TDqConfig;
-}
-
+namespace NProto {
+class TDqConfig;
+}
+
class IDqGateway : public TThrRefBase {
public:
using TPtr = TIntrusivePtr<IDqGateway>;
@@ -42,7 +42,7 @@ public:
using TUploadList = THashSet<TFileResource, TFileResourceHash, TFileResourceEqual>;
- class TResult: public NCommon::TOperationResult {
+ class TResult: public NCommon::TOperationResult {
public:
TString Data;
bool Fallback = false;
@@ -64,11 +64,11 @@ public:
virtual void CloseSession(const TString& sessionId) = 0;
virtual NThreading::TFuture<TResult>
- ExecutePlan(const TString& sessionId, NDqs::IDqsExecutionPlanner& plan, const TVector<TString>& columns,
- const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& graphParams,
- const TDqSettings::TPtr& settings,
- const TDqProgressWriter& progressWriter, const THashMap<TString, TString>& modulesMapping,
- bool discard) = 0;
+ ExecutePlan(const TString& sessionId, NDqs::IDqsExecutionPlanner& plan, const TVector<TString>& columns,
+ const THashMap<TString, TString>& secureParams, const THashMap<TString, TString>& graphParams,
+ const TDqSettings::TPtr& settings,
+ const TDqProgressWriter& progressWriter, const THashMap<TString, TString>& modulesMapping,
+ bool discard) = 0;
virtual TString GetVanillaJobPath() {
return "";
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp
index 69788c5c76..e4c0dd9e1a 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_provider.cpp
@@ -15,7 +15,7 @@ namespace NYql {
TDataProviderInitializer GetDqDataProviderInitializer(
TExecTransformerFactory execTransformerFactory,
const IDqGateway::TPtr& dqGateway,
- NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
+ NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
const IMetricsRegistryPtr& metrics,
const TFileStoragePtr& fileStorage,
bool externalUser)
@@ -32,11 +32,11 @@ TDataProviderInitializer GetDqDataProviderInitializer(
) {
Y_UNUSED(userName);
- TDqStatePtr state = MakeIntrusive<TDqState>(
+ TDqStatePtr state = MakeIntrusive<TDqState>(
dqGateway, // nullptr for yqlrun
gatewaysConfig,
functionRegistry,
- compFactory,
+ compFactory,
randomProvider,
typeCtx.Get(),
progressWriter,
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_provider.h b/ydb/library/yql/providers/dq/provider/yql_dq_provider.h
index 6e4aff4ee9..18480d8193 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_provider.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_provider.h
@@ -18,7 +18,7 @@ using TExecTransformerFactory = std::function<IGraphTransformer*(const TDqStateP
TDataProviderInitializer GetDqDataProviderInitializer(
TExecTransformerFactory execTransformerFactory,
const IDqGateway::TPtr& dqGateway,
- NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
+ NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
const IMetricsRegistryPtr& metrics,
const TFileStoragePtr& fileStorage,
bool externalUser = false);
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp b/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp
index 90e8af759c..a437f98223 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_recapture.cpp
@@ -1,6 +1,6 @@
-#include "yql_dq_state.h"
-#include "yql_dq_provider.h"
-
+#include "yql_dq_state.h"
+#include "yql_dq_provider.h"
+
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
#include <ydb/library/yql/providers/dq/expr_nodes/dqs_expr_nodes.h>
#include <ydb/library/yql/providers/dq/interface/yql_dq_integration.h>
@@ -12,39 +12,39 @@
#include <ydb/library/yql/ast/yql_expr.h>
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/dq/opt/dq_opt.h>
-
+
#include <util/generic/scope.h>
-namespace NYql {
-
-using namespace NNodes;
-
-namespace {
-
+namespace NYql {
+
+using namespace NNodes;
+
+namespace {
+
const THashSet<TStringBuf> VALID_SOURCES = {DqProviderName, ConfigProviderName, YtProviderName, ClickHouseProviderName, YdbProviderName};
const THashSet<TStringBuf> VALID_SINKS = {ResultProviderName, YtProviderName};
const THashSet<TStringBuf> UNSUPPORTED_CALLABLE = { TCoForwardList::CallableName() };
-
-}
-
+
+}
+
namespace NDq {
bool CheckJoinColumns(const TExprBase& node);
bool CheckJoinLinkSettings(const TExprBase& node);
} // namespace NDq
-class TDqsRecaptureTransformer : public TSyncTransformerBase {
-public:
- TDqsRecaptureTransformer(TDqStatePtr state)
- : State_(state)
- {
- }
-
- TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
- output = input;
- if (ctx.Step.IsDone(TExprStep::Recapture)) {
- return TStatus::Ok;
- }
-
+class TDqsRecaptureTransformer : public TSyncTransformerBase {
+public:
+ TDqsRecaptureTransformer(TDqStatePtr state)
+ : State_(state)
+ {
+ }
+
+ TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
+ output = input;
+ if (ctx.Step.IsDone(TExprStep::Recapture)) {
+ return TStatus::Ok;
+ }
+
Y_SCOPE_EXIT(&) {
FlushStatistics();
};
@@ -58,7 +58,7 @@ public:
Statistics_["DqForce"]++;
}
- if (!State_->TypeCtx->ForceDq) {
+ if (!State_->TypeCtx->ForceDq) {
if (!State_->Settings->AnalyzeQuery.Get().GetOrElse(false)) {
Statistics_["DqAnalyzerOff"]++;
}
@@ -68,17 +68,17 @@ public:
}
if (State_->TypeCtx->PureResultDataSource != DqProviderName || !State_->Settings->AnalyzeQuery.Get().GetOrElse(false)) {
- return TStatus::Ok;
- }
-
+ return TStatus::Ok;
+ }
+
Statistics_["DqAnalyzerOn"]++;
- ui64 dataSize = 0;
- bool good = true;
+ ui64 dataSize = 0;
+ bool good = true;
bool hasJoin = false;
- TNodeSet visited;
+ TNodeSet visited;
Scan(*input, ctx, good, dataSize, visited, hasJoin);
-
+
if (good) {
Statistics_["DqAnalyzerOk"]++;
} else {
@@ -91,110 +91,110 @@ public:
if (!good || (hasJoin && dataSize > State_->Settings->MaxDataSizePerQuery.Get().GetOrElse(10_GB))) {
YQL_LOG(DEBUG) << "good: " << good << " hasJoin: " << hasJoin << " dataSize: " << dataSize;
- return TStatus::Ok;
- }
- }
-
+ return TStatus::Ok;
+ }
+ }
+
State_->TypeCtx->DqFallbackPolicy = State_->Settings->FallbackPolicy.Get().GetOrElse("default");
- auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) {
- if (auto maybeRead = TMaybeNode<TCoRight>(node).Input()) {
- if (maybeRead.Raw()->ChildrenSize() > 1 && TCoDataSource::Match(maybeRead.Raw()->Child(1))) {
- auto dataSourceName = maybeRead.Raw()->Child(1)->Child(0)->Content();
+ auto status = OptimizeExpr(input, output, [&](const TExprNode::TPtr& node, TExprContext& ctx) {
+ if (auto maybeRead = TMaybeNode<TCoRight>(node).Input()) {
+ if (maybeRead.Raw()->ChildrenSize() > 1 && TCoDataSource::Match(maybeRead.Raw()->Child(1))) {
+ auto dataSourceName = maybeRead.Raw()->Child(1)->Child(0)->Content();
auto dataSource = State_->TypeCtx->DataSourceMap.FindPtr(dataSourceName);
YQL_ENSURE(dataSource);
if (auto dqIntegration = (*dataSource)->GetDqIntegration()) {
auto newRead = dqIntegration->WrapRead(*State_->Settings, maybeRead.Cast().Ptr(), ctx);
- if (newRead.Get() != maybeRead.Raw()) {
- return newRead;
- }
- }
- }
- }
-
- return node;
- }, ctx, TOptimizeExprSettings{State_->TypeCtx});
-
- if (input != output) {
+ if (newRead.Get() != maybeRead.Raw()) {
+ return newRead;
+ }
+ }
+ }
+ }
+
+ return node;
+ }, ctx, TOptimizeExprSettings{State_->TypeCtx});
+
+ if (input != output) {
// TODO: Add before/after recapture transformers
State_->TypeCtx->DqCaptured = true;
- // TODO: drop this after implementing DQS ConstraintTransformer
- State_->TypeCtx->ExpectedConstraints.clear();
- }
- return status;
- }
-
- void Rewind() final {
- }
-
-private:
+ // TODO: drop this after implementing DQS ConstraintTransformer
+ State_->TypeCtx->ExpectedConstraints.clear();
+ }
+ return status;
+ }
+
+ void Rewind() final {
+ }
+
+private:
void AddInfo(TExprContext& ctx, const TString& message) const {
YQL_LOG(DEBUG) << message;
TIssue info("DQ cannot execute the query. Cause: " + message);
info.Severity = TSeverityIds::S_INFO;
- ctx.IssueManager.RaiseIssue(info);
+ ctx.IssueManager.RaiseIssue(info);
}
- void Scan(const TExprNode& node, TExprContext& ctx, bool& good, ui64& dataSize, TNodeSet& visited, bool& hasJoin) const {
- if (!visited.insert(&node).second) {
- return;
- }
-
+ void Scan(const TExprNode& node, TExprContext& ctx, bool& good, ui64& dataSize, TNodeSet& visited, bool& hasJoin) const {
+ if (!visited.insert(&node).second) {
+ return;
+ }
+
TExprBase expr(&node);
if (TMaybeNode<TCoEquiJoin>(&node)) {
hasJoin = true;
}
- if (TCoCommit::Match(&node)) {
- for (size_t i = 0; i != node.ChildrenSize() && good; ++i) {
- if (i != TCoCommit::idx_DataSink) {
+ if (TCoCommit::Match(&node)) {
+ for (size_t i = 0; i != node.ChildrenSize() && good; ++i) {
+ if (i != TCoCommit::idx_DataSink) {
Scan(*node.Child(i), ctx, good, dataSize, visited, hasJoin);
- }
- }
- } else if (node.IsCallable(UNSUPPORTED_CALLABLE)) {
- AddInfo(ctx, TStringBuilder() << "unsupported callable '" << node.Content() << "'");
- good = false;
+ }
+ }
+ } else if (node.IsCallable(UNSUPPORTED_CALLABLE)) {
+ AddInfo(ctx, TStringBuilder() << "unsupported callable '" << node.Content() << "'");
+ good = false;
} else if (node.IsCallable(TCoCollect::CallableName())) {
- if (ETypeAnnotationKind::List != node.Head().GetTypeAnn()->GetKind()) {
- AddInfo(ctx, TStringBuilder() << "unsupported callable '" << node.Content() << "' over stream/flow");
- good = false;
- }
- } else if (auto datasource = TMaybeNode<TCoDataSource>(&node).Category()) {
- if (!VALID_SOURCES.contains(datasource.Cast().Value())) {
+ if (ETypeAnnotationKind::List != node.Head().GetTypeAnn()->GetKind()) {
+ AddInfo(ctx, TStringBuilder() << "unsupported callable '" << node.Content() << "' over stream/flow");
+ good = false;
+ }
+ } else if (auto datasource = TMaybeNode<TCoDataSource>(&node).Category()) {
+ if (!VALID_SOURCES.contains(datasource.Cast().Value())) {
AddInfo(ctx, TStringBuilder() << "source '" << datasource.Cast().Value() << "' is not supported by DQ");
- good = false;
- }
- } else if (auto datasink = TMaybeNode<TCoDataSink>(&node).Category()) {
- if (!VALID_SINKS.contains(datasink.Cast().Value())) {
+ good = false;
+ }
+ } else if (auto datasink = TMaybeNode<TCoDataSink>(&node).Category()) {
+ if (!VALID_SINKS.contains(datasink.Cast().Value())) {
AddInfo(ctx, TStringBuilder() << "sink '" << datasink.Cast().Value() << "' is not supported by DQ");
- good = false;
- }
+ good = false;
+ }
} else if (TMaybeNode<TCoEquiJoin>(&node) && !NDq::CheckJoinColumns(expr)) {
AddInfo(ctx, TStringBuilder() << "unsupported join column");
good = false;
} else if (TMaybeNode<TCoEquiJoin>(&node) && !NDq::CheckJoinLinkSettings(expr)) {
AddInfo(ctx, TStringBuilder() << "unsupported join any");
good = false;
- } else if (node.ChildrenSize() > 1 && TCoDataSource::Match(node.Child(1))) {
- auto dataSourceName = node.Child(1)->Child(0)->Content();
- if (dataSourceName != DqProviderName && !node.IsCallable(ConfigureName)) {
- auto datasource = State_->TypeCtx->DataSourceMap.FindPtr(dataSourceName);
- YQL_ENSURE(datasource);
- if (auto dqIntegration = (*datasource)->GetDqIntegration()) {
+ } else if (node.ChildrenSize() > 1 && TCoDataSource::Match(node.Child(1))) {
+ auto dataSourceName = node.Child(1)->Child(0)->Content();
+ if (dataSourceName != DqProviderName && !node.IsCallable(ConfigureName)) {
+ auto datasource = State_->TypeCtx->DataSourceMap.FindPtr(dataSourceName);
+ YQL_ENSURE(datasource);
+ if (auto dqIntegration = (*datasource)->GetDqIntegration()) {
if (auto size = dqIntegration->CanRead(*State_->Settings, node, ctx, /*skipIssues = */ false)) {
- dataSize += *size;
- } else {
- good = false;
- }
- } else {
- AddInfo(ctx, TStringBuilder() << "source '" << dataSourceName << "' is not supported by DQ");
- good = false;
- }
- }
-
- if (good) {
+ dataSize += *size;
+ } else {
+ good = false;
+ }
+ } else {
+ AddInfo(ctx, TStringBuilder() << "source '" << dataSourceName << "' is not supported by DQ");
+ good = false;
+ }
+ }
+
+ if (good) {
Scan(node.Head(), ctx,good, dataSize, visited, hasJoin);
- }
+ }
} else if (node.GetTypeAnn()->GetKind() == ETypeAnnotationKind::World
&& !TCoCommit::Match(&node)
&& node.ChildrenSize() > 1
@@ -203,12 +203,12 @@ private:
auto dataSink = State_->TypeCtx->DataSinkMap.FindPtr(dataSinkName);
YQL_ENSURE(dataSink);
if (auto dqIntegration = dataSink->Get()->GetDqIntegration()) {
- if (auto canWrite = dqIntegration->CanWrite(*State_->Settings, node, ctx)) {
- if (!canWrite.GetRef()) {
- good = false;
- } else if (!State_->Settings->EnableInsert.Get().GetOrElse(false)) {
- AddInfo(ctx, TStringBuilder() << "'insert' support is disabled. Use PRAGMA dq.EnableInsert to explicitly enable it");
+ if (auto canWrite = dqIntegration->CanWrite(*State_->Settings, node, ctx)) {
+ if (!canWrite.GetRef()) {
good = false;
+ } else if (!State_->Settings->EnableInsert.Get().GetOrElse(false)) {
+ AddInfo(ctx, TStringBuilder() << "'insert' support is disabled. Use PRAGMA dq.EnableInsert to explicitly enable it");
+ good = false;
}
}
}
@@ -218,26 +218,26 @@ private:
}
}
}
- else if (!State_->TypeCtx->UdfSupportsYield && TCoScriptUdf::Match(&node)) {
- if (IsCallableTypeHasStreams(node.GetTypeAnn()->Cast<TCallableExprType>())) {
- AddInfo(ctx, TStringBuilder() << "script udf with streams");
- good = false;
- }
- if (good) {
- for (size_t i = 0; i != node.ChildrenSize() && good; ++i) {
- Scan(*node.Child(i), ctx, good, dataSize, visited, hasJoin);
- }
- }
- }
+ else if (!State_->TypeCtx->UdfSupportsYield && TCoScriptUdf::Match(&node)) {
+ if (IsCallableTypeHasStreams(node.GetTypeAnn()->Cast<TCallableExprType>())) {
+ AddInfo(ctx, TStringBuilder() << "script udf with streams");
+ good = false;
+ }
+ if (good) {
+ for (size_t i = 0; i != node.ChildrenSize() && good; ++i) {
+ Scan(*node.Child(i), ctx, good, dataSize, visited, hasJoin);
+ }
+ }
+ }
else {
- for (size_t i = 0; i != node.ChildrenSize() && good; ++i) {
+ for (size_t i = 0; i != node.ChildrenSize() && good; ++i) {
Scan(*node.Child(i), ctx, good, dataSize, visited, hasJoin);
- }
- }
- }
-
-private:
- TDqStatePtr State_;
+ }
+ }
+ }
+
+private:
+ TDqStatePtr State_;
THashMap<TString, int> Statistics_;
@@ -254,10 +254,10 @@ private:
State_->Statistics[State_->MetricId++] = statistics;
}
}
-};
-
-THolder<IGraphTransformer> CreateDqsRecaptureTransformer(TDqStatePtr state) {
+};
+
+THolder<IGraphTransformer> CreateDqsRecaptureTransformer(TDqStatePtr state) {
return THolder(new TDqsRecaptureTransformer(state));
-}
-
-} // NYql
+}
+
+} // NYql
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_recapture.h b/ydb/library/yql/providers/dq/provider/yql_dq_recapture.h
index 4db8f3a4b2..7506ae7bd0 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_recapture.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_recapture.h
@@ -1,13 +1,13 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/core/yql_graph_transformer.h>
-
-#include <util/generic/ptr.h>
-
-namespace NYql {
- struct TDqState;
- using TDqStatePtr = TIntrusivePtr<TDqState>;
-
- THolder<IGraphTransformer> CreateDqsRecaptureTransformer(TDqStatePtr state);
-
-} // namespace NYql
+
+#include <util/generic/ptr.h>
+
+namespace NYql {
+ struct TDqState;
+ using TDqStatePtr = TIntrusivePtr<TDqState>;
+
+ THolder<IGraphTransformer> CreateDqsRecaptureTransformer(TDqStatePtr state);
+
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/provider/yql_dq_state.h b/ydb/library/yql/providers/dq/provider/yql_dq_state.h
index a3e54dcb11..f576a02e4c 100644
--- a/ydb/library/yql/providers/dq/provider/yql_dq_state.h
+++ b/ydb/library/yql/providers/dq/provider/yql_dq_state.h
@@ -7,17 +7,17 @@
#include <ydb/library/yql/providers/dq/common/yql_dq_settings.h>
#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
-#include <util/generic/ptr.h>
-
+#include <util/generic/ptr.h>
+
namespace NYql {
using namespace NDqs; // TODO: remove this namespace;
-struct TDqState: public TThrRefBase {
+struct TDqState: public TThrRefBase {
IDqGateway::TPtr DqGateway;
const TGatewaysConfig* GatewaysConfig;
const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry;
- NKikimr::NMiniKQL::TComputationNodeFactory ComputationFactory;
+ NKikimr::NMiniKQL::TComputationNodeFactory ComputationFactory;
TIntrusivePtr<IRandomProvider> RandomProvider;
TTypeAnnotationContext* TypeCtx;
const TOperationProgressWriter ProgressWriter;
@@ -39,7 +39,7 @@ struct TDqState: public TThrRefBase {
const IDqGateway::TPtr& dqGateway,
const TGatewaysConfig* gatewaysConfig,
const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
+ NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
const TIntrusivePtr<IRandomProvider>& randomProvider,
TTypeAnnotationContext* typeCtx,
const TOperationProgressWriter& progressWriter,
@@ -53,7 +53,7 @@ struct TDqState: public TThrRefBase {
: DqGateway(dqGateway)
, GatewaysConfig(gatewaysConfig)
, FunctionRegistry(functionRegistry)
- , ComputationFactory(compFactory)
+ , ComputationFactory(compFactory)
, RandomProvider(randomProvider)
, TypeCtx(typeCtx)
, ProgressWriter(progressWriter)
@@ -67,6 +67,6 @@ struct TDqState: public TThrRefBase {
{ }
};
-using TDqStatePtr = TIntrusivePtr<TDqState>;
+using TDqStatePtr = TIntrusivePtr<TDqState>;
} // namespace
diff --git a/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp b/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp
index 9b41920b17..34bcbb9af4 100644
--- a/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp
+++ b/ydb/library/yql/providers/dq/runtime/task_command_executor.cpp
@@ -1,5 +1,5 @@
-#include "task_command_executor.h"
-
+#include "task_command_executor.h"
+
#include <ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h>
#include <ydb/library/yql/providers/dq/counters/counters.h>
#include <ydb/library/yql/providers/dq/common/yql_dq_settings.h>
@@ -89,12 +89,12 @@ void ToProto(T* s1, const NDq::TDqSinkStats* ss)
class TTaskCommandExecutor {
public:
TTaskCommandExecutor(NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, NKikimr::NMiniKQL::IStatsRegistry* jobStats, bool terminateOnError)
- : ComputationFactory(std::move(compFactory))
- , TaskTransformFactory(std::move(taskTransformFactory))
+ : ComputationFactory(std::move(compFactory))
+ , TaskTransformFactory(std::move(taskTransformFactory))
, JobStats(std::move(jobStats))
, TerminateOnError(terminateOnError)
- {
- }
+ {
+ }
void UpdateStats() {
if (!Runner) {
@@ -294,7 +294,7 @@ public:
NDqProto::TData data;
data.Load(&input);
- auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
+ auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
channel->Push(std::move(data));
UpdateInputChannelStats(channelId);
break;
@@ -307,7 +307,7 @@ public:
NDqProto::TSourcePushRequest request;
request.Load(&input);
- auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
+ auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
auto transportVersion = NDqProto::EDataTransportVersion::DATA_TRANSPORT_VERSION_UNSPECIFIED;
switch (request.GetData().GetTransportVersion()) {
case 10000: {
@@ -421,7 +421,7 @@ public:
}
case NDqProto::TCommandHeader::POP: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
+ auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
Y_ENSURE(taskId == Runner->GetTaskId());
@@ -450,13 +450,13 @@ public:
auto guard = Runner->BindAllocator(DqConfiguration->MemoryLimit.Get().GetOrElse(0));
Y_ENSURE(taskId == Runner->GetTaskId());
- try {
- NDqProto::TRunResponse response;
- auto status = Runner->Run();
- response.SetResult(static_cast<ui32>(status));
+ try {
+ NDqProto::TRunResponse response;
+ auto status = Runner->Run();
+ response.SetResult(static_cast<ui32>(status));
UpdateStats(response);
- response.Save(&output);
- } catch (const NKikimr::TMemoryLimitExceededException& ex) {
+ response.Save(&output);
+ } catch (const NKikimr::TMemoryLimitExceededException& ex) {
throw yexception() << "DQ computation exceeds the memory limit " << DqConfiguration->MemoryLimit.Get().GetOrElse(0) << ". Try to increase the limit using PRAGMA dq.MemoryLimit";
}
break;
@@ -472,17 +472,17 @@ public:
});
request.GetTask().GetMeta().UnpackTo(&taskMeta);
- try {
- Prepare(request.GetTask(), taskMeta, output);
- } catch (const NKikimr::TMemoryLimitExceededException& ex) {
- throw yexception() << "DQ computation exceeds the memory limit " << DqConfiguration->MemoryLimit.Get().GetOrElse(0) << ". Try to increase the limit using PRAGMA dq.MemoryLimit";
- }
-
+ try {
+ Prepare(request.GetTask(), taskMeta, output);
+ } catch (const NKikimr::TMemoryLimitExceededException& ex) {
+ throw yexception() << "DQ computation exceeds the memory limit " << DqConfiguration->MemoryLimit.Get().GetOrElse(0) << ". Try to increase the limit using PRAGMA dq.MemoryLimit";
+ }
+
break;
}
case NDqProto::TCommandHeader::GET_INPUT_TYPE: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
+ auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
Y_ENSURE(taskId == Runner->GetTaskId());
auto channel = Runner->GetInputChannel(channelId);
@@ -496,7 +496,7 @@ public:
}
case NDqProto::TCommandHeader::GET_SOURCE_TYPE: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
+ auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
Y_ENSURE(taskId == Runner->GetTaskId());
auto source = Runner->GetSource(channelId);
@@ -621,7 +621,7 @@ public:
NDqProto::TSinkPopRequest request;
request.Load(&input);
- auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
+ auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
NKikimr::NMiniKQL::TUnboxedValueVector batch;
auto sink = Runner->GetSink(channelId);
auto* outputType = sink->GetOutputType();
@@ -648,7 +648,7 @@ public:
}
case NDqProto::TCommandHeader::SINK_OUTPUT_TYPE: {
Y_ENSURE(header.GetVersion() <= CurrentProtocolVersion);
- auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
+ auto guard = Runner->BindAllocator(0); // Explicitly reset memory limit
Y_ENSURE(taskId == Runner->GetTaskId());
auto outputType = Runner->GetSink(channelId)->GetOutputType();
@@ -752,7 +752,7 @@ public:
Runner = MakeDqTaskRunner(Ctx, settings, nullptr);
});
- auto guard = Runner->BindAllocator(DqConfiguration->MemoryLimit.Get().GetOrElse(0));
+ auto guard = Runner->BindAllocator(DqConfiguration->MemoryLimit.Get().GetOrElse(0));
QueryStat.Measure<void>("Prepare", [&]() {
NDq::TDqTaskRunnerMemoryLimits limits;
@@ -764,8 +764,8 @@ public:
result.Save(&output);
}
- NKikimr::NMiniKQL::TComputationNodeFactory ComputationFactory;
- TTaskTransformFactory TaskTransformFactory;
+ NKikimr::NMiniKQL::TComputationNodeFactory ComputationFactory;
+ TTaskTransformFactory TaskTransformFactory;
THashMap<TString, i64> CurrentJobStats;
NKikimr::NMiniKQL::IStatsRegistry* JobStats;
bool TerminateOnError;
diff --git a/ydb/library/yql/providers/dq/runtime/task_command_executor.h b/ydb/library/yql/providers/dq/runtime/task_command_executor.h
index 75f7c60177..35dc159ff9 100644
--- a/ydb/library/yql/providers/dq/runtime/task_command_executor.h
+++ b/ydb/library/yql/providers/dq/runtime/task_command_executor.h
@@ -1,13 +1,13 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/minikql/computation/mkql_computation_node.h>
#include <ydb/library/yql/minikql/mkql_stats_registry.h>
#include <ydb/library/yql/providers/dq/interface/yql_dq_task_transform.h>
-
-namespace NYql {
-namespace NTaskRunnerProxy {
-
+
+namespace NYql {
+namespace NTaskRunnerProxy {
+
int CreateTaskCommandExecutor(NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory, NKikimr::NMiniKQL::IStatsRegistry* jobStats, bool terminateOnError = false);
-
-} // namespace NTaskRunnerProxy
-} // namespace NYql
+
+} // namespace NTaskRunnerProxy
+} // namespace NYql
diff --git a/ydb/library/yql/providers/dq/service/grpc_service.cpp b/ydb/library/yql/providers/dq/service/grpc_service.cpp
index 65a9e500d9..2db2faa113 100644
--- a/ydb/library/yql/providers/dq/service/grpc_service.cpp
+++ b/ydb/library/yql/providers/dq/service/grpc_service.cpp
@@ -95,8 +95,8 @@ namespace NYql::NDqs {
}
void OnPoison() {
- YQL_LOG_CTX_SCOPE(TraceId);
- YQL_LOG(DEBUG) << __FUNCTION__ ;
+ YQL_LOG_CTX_SCOPE(TraceId);
+ YQL_LOG(DEBUG) << __FUNCTION__ ;
ReplyError(grpc::UNAVAILABLE, "Unexpected error");
*ClientDisconnectedCounter += 1;
}
@@ -106,7 +106,7 @@ namespace NYql::NDqs {
}
void DoBootstrap(const NActors::TActorContext& ctx) {
- YQL_LOG_CTX_SCOPE(TraceId);
+ YQL_LOG_CTX_SCOPE(TraceId);
if (!CtxSubscribed) {
auto selfId = ctx.SelfID;
auto* actorSystem = ctx.ExecutorThread.ActorSystem;
@@ -165,7 +165,7 @@ namespace NYql::NDqs {
*operation.Mutableissues() = result.GetIssues();
ResponseBuffer.SetTruncated(result.GetTruncated());
- Reply(Ydb::StatusIds::SUCCESS, result.GetIssues().size() > 0);
+ Reply(Ydb::StatusIds::SUCCESS, result.GetIssues().size() > 0);
}
virtual void DoRetry()
@@ -212,8 +212,8 @@ namespace NYql::NDqs {
this->PassAway();
}
- virtual void Reply(ui32 status, bool hasIssues) {
- Y_UNUSED(hasIssues);
+ virtual void Reply(ui32 status, bool hasIssues) {
+ Y_UNUSED(hasIssues);
Ctx->Reply(&ResponseBuffer, status);
this->PassAway();
}
@@ -264,7 +264,7 @@ namespace NYql::NDqs {
}
void DoRetry() override {
- YQL_LOG(DEBUG) << __FUNCTION__;
+ YQL_LOG(DEBUG) << __FUNCTION__;
SendEvent(NYql::NDqProto::EGraphExecutionEventType::FAIL, nullptr, [this](const auto& ev) {
if (ev->Get()->Record.GetErrorMessage()) {
TBase::ReplyError(grpc::UNAVAILABLE, ev->Get()->Record.GetErrorMessage());
@@ -276,15 +276,15 @@ namespace NYql::NDqs {
});
}
- void Reply(ui32 status, bool hasIssues) override {
- auto eventType = hasIssues
- ? NYql::NDqProto::EGraphExecutionEventType::FAIL
- : NYql::NDqProto::EGraphExecutionEventType::SUCCESS;
- SendEvent(eventType, nullptr, [this, status, hasIssues](const auto& ev) {
- if (!hasIssues && ev->Get()->Record.GetErrorMessage()) {
+ void Reply(ui32 status, bool hasIssues) override {
+ auto eventType = hasIssues
+ ? NYql::NDqProto::EGraphExecutionEventType::FAIL
+ : NYql::NDqProto::EGraphExecutionEventType::SUCCESS;
+ SendEvent(eventType, nullptr, [this, status, hasIssues](const auto& ev) {
+ if (!hasIssues && ev->Get()->Record.GetErrorMessage()) {
TBase::ReplyError(grpc::UNAVAILABLE, ev->Get()->Record.GetErrorMessage());
} else {
- TBase::Reply(status, hasIssues);
+ TBase::Reply(status, hasIssues);
}
});
}
@@ -297,60 +297,60 @@ namespace NYql::NDqs {
}
private:
- THolder<Yql::DqsProto::ExecuteGraphRequest> ModifiedRequest;
+ THolder<Yql::DqsProto::ExecuteGraphRequest> ModifiedRequest;
void DoPassAway() override {
- YQL_LOG(DEBUG) << __FUNCTION__;
+ YQL_LOG(DEBUG) << __FUNCTION__;
Send(GraphExecutionEventsActorId, new TEvents::TEvPoison());
TServiceProxyActor::DoPassAway();
}
- NDqProto::TGraphExecutionEvent::TExecuteGraphDescriptor SerializeGraphDescriptor() const {
+ NDqProto::TGraphExecutionEvent::TExecuteGraphDescriptor SerializeGraphDescriptor() const {
NDqProto::TGraphExecutionEvent::TExecuteGraphDescriptor result;
for (const auto& kv : Request->GetSecureParams()) {
result.MutableSecureParams()->MutableData()->insert(kv);
}
- for (const auto& kv : Request->GetGraphParams()) {
- result.MutableGraphParams()->MutableData()->insert(kv);
- }
-
+ for (const auto& kv : Request->GetGraphParams()) {
+ result.MutableGraphParams()->MutableData()->insert(kv);
+ }
+
return result;
}
void Bootstrap() override {
YQL_LOG(DEBUG) << "TServiceProxyActor::OnExecuteGraph";
- SendEvent(NYql::NDqProto::EGraphExecutionEventType::START, SerializeGraphDescriptor(), [this](const TEvGraphExecutionEvent::TPtr& ev) {
- if (ev->Get()->Record.GetErrorMessage()) {
- TBase::ReplyError(grpc::UNAVAILABLE, ev->Get()->Record.GetErrorMessage());
- } else {
- NDqProto::TGraphExecutionEvent::TMap params;
- ev->Get()->Record.GetMessage().UnpackTo(&params);
- FinishBootstrap(params);
- }
+ SendEvent(NYql::NDqProto::EGraphExecutionEventType::START, SerializeGraphDescriptor(), [this](const TEvGraphExecutionEvent::TPtr& ev) {
+ if (ev->Get()->Record.GetErrorMessage()) {
+ TBase::ReplyError(grpc::UNAVAILABLE, ev->Get()->Record.GetErrorMessage());
+ } else {
+ NDqProto::TGraphExecutionEvent::TMap params;
+ ev->Get()->Record.GetMessage().UnpackTo(&params);
+ FinishBootstrap(params);
+ }
});
}
- void MergeTaskMetas(const NDqProto::TGraphExecutionEvent::TMap& params) {
- if (!params.data().empty()) {
- for (size_t i = 0; i < Request->TaskSize(); ++i) {
- if (!ModifiedRequest) {
- ModifiedRequest.Reset(new Yql::DqsProto::ExecuteGraphRequest());
- ModifiedRequest->CopyFrom(*Request);
- }
+ void MergeTaskMetas(const NDqProto::TGraphExecutionEvent::TMap& params) {
+ if (!params.data().empty()) {
+ for (size_t i = 0; i < Request->TaskSize(); ++i) {
+ if (!ModifiedRequest) {
+ ModifiedRequest.Reset(new Yql::DqsProto::ExecuteGraphRequest());
+ ModifiedRequest->CopyFrom(*Request);
+ }
- auto* task = ModifiedRequest->MutableTask(i);
+ auto* task = ModifiedRequest->MutableTask(i);
- Yql::DqsProto::TTaskMeta taskMeta;
- task->GetMeta().UnpackTo(&taskMeta);
+ Yql::DqsProto::TTaskMeta taskMeta;
+ task->GetMeta().UnpackTo(&taskMeta);
- for (const auto&[key, value] : params.data()) {
- (*taskMeta.MutableTaskParams())[key] = value;
- }
+ for (const auto&[key, value] : params.data()) {
+ (*taskMeta.MutableTaskParams())[key] = value;
+ }
- task->MutableMeta()->PackFrom(taskMeta);
+ task->MutableMeta()->PackFrom(taskMeta);
}
}
@@ -359,9 +359,9 @@ namespace NYql::NDqs {
}
}
- void FinishBootstrap(const NDqProto::TGraphExecutionEvent::TMap& params) {
- YQL_LOG(DEBUG) << __FUNCTION__;
- MergeTaskMetas(params);
+ void FinishBootstrap(const NDqProto::TGraphExecutionEvent::TMap& params) {
+ YQL_LOG(DEBUG) << __FUNCTION__;
+ MergeTaskMetas(params);
auto executerId = RegisterChild(NDq::MakeDqExecuter(MakeWorkerManagerActorID(SelfId().NodeId()), SelfId(), TraceId, Username, Settings, Counters, RequestStartTime));
@@ -380,14 +380,14 @@ namespace NYql::NDqs {
secureParams[x.first] = x.second;
}
auto resultId = RegisterChild(NExecutionHelpers::MakeResultAggregator(
- columns,
- executerId,
- TraceId,
- secureParams,
- Settings,
- Request->GetResultType(),
- Request->GetDiscard(),
- GraphExecutionEventsActorId).Release());
+ columns,
+ executerId,
+ TraceId,
+ secureParams,
+ Settings,
+ Request->GetResultType(),
+ Request->GetDiscard(),
+ GraphExecutionEventsActorId).Release());
auto controlId = Settings->EnableComputeActor.Get().GetOrElse(false) == false ? resultId
: RegisterChild(NYql::MakeTaskController(TraceId, executerId, resultId, Settings, NYq::NCommon::TServiceCounters(Counters, nullptr, "")).Release());
Send(executerId, MakeHolder<TEvGraphRequest>(
@@ -404,8 +404,8 @@ namespace NYql::NDqs {
record.MutableMessage()->PackFrom(payload);
}
Send(GraphExecutionEventsActorId, new TEvGraphExecutionEvent(record));
- Synchronize<TEvGraphExecutionEvent>([callback, traceId = TraceId](TEvGraphExecutionEvent::TPtr& ev) {
- YQL_LOG_CTX_SCOPE(traceId);
+ Synchronize<TEvGraphExecutionEvent>([callback, traceId = TraceId](TEvGraphExecutionEvent::TPtr& ev) {
+ YQL_LOG_CTX_SCOPE(traceId);
Y_VERIFY(ev->Get()->Record.GetEventType() == NYql::NDqProto::EGraphExecutionEventType::SYNC);
callback(ev);
});
@@ -459,7 +459,7 @@ namespace NYql::NDqs {
CQ = cq;
- using TDqTaskPreprocessorCollection = std::vector<NYql::IDqTaskPreprocessor::TPtr>;
+ using TDqTaskPreprocessorCollection = std::vector<NYql::IDqTaskPreprocessor::TPtr>;
ADD_REQUEST(ExecuteGraph, ExecuteGraphRequest, ExecuteGraphResponse, {
TGuard<TMutex> lock(Mutex);
@@ -479,14 +479,14 @@ namespace NYql::NDqs {
}
TDqTaskPreprocessorCollection taskPreprocessors;
- for (const auto& factory: DqTaskPreprocessorFactories) {
- taskPreprocessors.push_back(factory());
+ for (const auto& factory: DqTaskPreprocessorFactories) {
+ taskPreprocessors.push_back(factory());
}
- auto graphExecutionEventsActorId = ActorSystem.Register(NDqs::MakeGraphExecutionEventsActor(request->GetSession(), std::move(taskPreprocessors)));
+ auto graphExecutionEventsActorId = ActorSystem.Register(NDqs::MakeGraphExecutionEventsActor(request->GetSession(), std::move(taskPreprocessors)));
RunningRequests++;
- auto actor = MakeHolder<TExecuteGraphProxyActor>(ctx, Counters, request->GetSession(), session->GetUsername(), graphExecutionEventsActorId);
+ auto actor = MakeHolder<TExecuteGraphProxyActor>(ctx, Counters, request->GetSession(), session->GetUsername(), graphExecutionEventsActorId);
auto future = actor->GetFuture();
auto actorId = ActorSystem.Register(actor.Release());
future.Apply([session, actorId, this] (const TFuture<void>&) mutable {
diff --git a/ydb/library/yql/providers/dq/task_runner/file_cache.cpp b/ydb/library/yql/providers/dq/task_runner/file_cache.cpp
index 1608f3f31c..290ecf696b 100644
--- a/ydb/library/yql/providers/dq/task_runner/file_cache.cpp
+++ b/ydb/library/yql/providers/dq/task_runner/file_cache.cpp
@@ -1 +1 @@
-#include "file_cache.h"
+#include "file_cache.h"
diff --git a/ydb/library/yql/providers/dq/task_runner/file_cache.h b/ydb/library/yql/providers/dq/task_runner/file_cache.h
index 33e707334f..b3e252cfc3 100644
--- a/ydb/library/yql/providers/dq/task_runner/file_cache.h
+++ b/ydb/library/yql/providers/dq/task_runner/file_cache.h
@@ -6,8 +6,8 @@
#include <util/generic/maybe.h>
#include <util/system/mutex.h>
-#include <atomic>
-
+#include <atomic>
+
namespace NYql {
class IFileCache: public TSimpleRefCount<IFileCache> {
diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp
index ab8769f44e..127b7cf06b 100644
--- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp
+++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.cpp
@@ -1,5 +1,5 @@
-#include "tasks_runner_local.h"
-#include "file_cache.h"
+#include "tasks_runner_local.h"
+#include "file_cache.h"
#include <ydb/library/yql/providers/dq/counters/counters.h>
#include <ydb/library/yql/dq/runtime/dq_input_channel.h>
@@ -13,16 +13,16 @@
#include <ydb/library/yql/utils/backtrace/backtrace.h>
#include <ydb/library/yql/utils/yql_panic.h>
-#include <library/cpp/yson/node/node.h>
-#include <library/cpp/yson/node/node_io.h>
-#include <library/cpp/svnversion/svnversion.h>
-
+#include <library/cpp/yson/node/node.h>
+#include <library/cpp/yson/node/node_io.h>
+#include <library/cpp/svnversion/svnversion.h>
+
#include <util/system/env.h>
#include <util/stream/file.h>
-#include <util/generic/size_literals.h>
+#include <util/generic/size_literals.h>
-namespace NYql::NTaskRunnerProxy {
+namespace NYql::NTaskRunnerProxy {
using namespace NKikimr;
using namespace NDq;
@@ -206,7 +206,7 @@ private:
class TAbstractFactory: public IProxyFactory {
public:
- TAbstractFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory)
+ TAbstractFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry, NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory)
: DeterministicMode(!!GetEnv("YQL_DETERMINISTIC_MODE"))
, RandomProvider(
DeterministicMode
@@ -217,10 +217,10 @@ public:
? CreateDeterministicTimeProvider(10000000)
: CreateDefaultTimeProvider())
, FunctionRegistry(functionRegistry)
- , TaskTransformFactory(std::move(taskTransformFactory))
+ , TaskTransformFactory(std::move(taskTransformFactory))
{
ExecutionContext.FuncRegistry = FunctionRegistry;
- ExecutionContext.ComputationFactory = compFactory;
+ ExecutionContext.ComputationFactory = compFactory;
ExecutionContext.RandomProvider = RandomProvider.Get();
ExecutionContext.TimeProvider = TimeProvider.Get();
}
@@ -230,20 +230,20 @@ protected:
TIntrusivePtr<IRandomProvider> RandomProvider;
TIntrusivePtr<ITimeProvider> TimeProvider;
const NKikimr::NMiniKQL::IFunctionRegistry* FunctionRegistry;
- TTaskTransformFactory TaskTransformFactory;
+ TTaskTransformFactory TaskTransformFactory;
NDq::TDqTaskRunnerContext ExecutionContext;
};
-/*______________________________________________________________________________________________*/
-
+/*______________________________________________________________________________________________*/
+
class TLocalFactory: public TAbstractFactory {
public:
TLocalFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
TTaskTransformFactory taskTransformFactory,
bool terminateOnError)
- : TAbstractFactory(functionRegistry, compFactory, taskTransformFactory)
+ : TAbstractFactory(functionRegistry, compFactory, taskTransformFactory)
, TerminateOnError(terminateOnError)
{ }
@@ -253,7 +253,7 @@ public:
TIntrusivePtr<NDq::IDqTaskRunner> Get(const NDqProto::TDqTask& task, const TString& traceId) override {
Y_UNUSED(traceId);
- NDq::TDqTaskRunnerSettings settings;
+ NDq::TDqTaskRunnerSettings settings;
settings.TerminateOnError = TerminateOnError;
settings.CollectBasicStats = true;
settings.CollectProfileStats = true;
@@ -267,13 +267,13 @@ public:
settings.OptLLVM = s.GetValue();
}
for (const auto& x : taskMeta.GetSecureParams()) {
- settings.SecureParams[x.first] = x.second;
- }
-
+ settings.SecureParams[x.first] = x.second;
+ }
+
for (const auto& x : taskMeta.GetTaskParams()) {
settings.TaskParams[x.first] = x.second;
- }
- auto ctx = ExecutionContext;
+ }
+ auto ctx = ExecutionContext;
ctx.FuncProvider = TaskTransformFactory(settings.TaskParams, ctx.FuncRegistry);
return MakeDqTaskRunner(ctx, settings, { });
}
@@ -282,11 +282,11 @@ private:
const bool TerminateOnError;
};
-IProxyFactory::TPtr CreateFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
- NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
+IProxyFactory::TPtr CreateFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
+ NKikimr::NMiniKQL::TComputationNodeFactory compFactory,
TTaskTransformFactory taskTransformFactory, bool terminateOnError)
{
return new TLocalFactory(functionRegistry, compFactory, taskTransformFactory, terminateOnError);
}
-} // namespace NYql::NTaskRunnerProxy
+} // namespace NYql::NTaskRunnerProxy
diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h
index 1702041a74..7c790aa8d9 100644
--- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h
+++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_local.h
@@ -1,14 +1,14 @@
#pragma once
-#include "tasks_runner_proxy.h"
+#include "tasks_runner_proxy.h"
#include <ydb/library/yql/providers/dq/interface/yql_dq_task_transform.h>
-namespace NYql::NTaskRunnerProxy {
+namespace NYql::NTaskRunnerProxy {
-IProxyFactory::TPtr CreateFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
+IProxyFactory::TPtr CreateFactory(const NKikimr::NMiniKQL::IFunctionRegistry* functionRegistry,
NKikimr::NMiniKQL::TComputationNodeFactory compFactory, TTaskTransformFactory taskTransformFactory,
bool terminateOnError
);
-} // namespace NYql::NTaskRunnerProxy
+} // namespace NYql::NTaskRunnerProxy
diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp
index cdcbd32ef5..312d75f655 100644
--- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp
+++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.cpp
@@ -1,4 +1,4 @@
-#include "tasks_runner_pipe.h"
+#include "tasks_runner_pipe.h"
#include <ydb/library/yql/dq/runtime/dq_input_channel.h>
#include <ydb/library/yql/dq/runtime/dq_output_channel.h>
@@ -13,10 +13,10 @@
#include <ydb/library/yql/providers/dq/common/yql_dq_settings.h>
-#include <library/cpp/yson/node/node.h>
-#include <library/cpp/yson/node/node_io.h>
-#include <library/cpp/svnversion/svnversion.h>
-
+#include <library/cpp/yson/node/node.h>
+#include <library/cpp/yson/node/node_io.h>
+#include <library/cpp/svnversion/svnversion.h>
+
#include <library/cpp/threading/task_scheduler/task_scheduler.h>
#include <util/system/shellcommand.h>
@@ -24,17 +24,17 @@
#include <util/system/fs.h>
#include <util/stream/file.h>
#include <util/stream/pipe.h>
-#include <util/generic/size_literals.h>
-#include <util/string/cast.h>
+#include <util/generic/size_literals.h>
+#include <util/string/cast.h>
-namespace NYql::NTaskRunnerProxy {
+namespace NYql::NTaskRunnerProxy {
const TString WorkingDirectoryParamName = "TaskRunnerProxy.WorkingDirectory";
const TString WorkingDirectoryDontInitParamName = "TaskRunnerProxy.WorkingDirectoryDontInit";
const TString UseMetaParamName = "TaskRunnerProxy.UseMeta"; // COMPAT(aozeritsky)
-using namespace NKikimr;
-using namespace NDq;
+using namespace NKikimr;
+using namespace NDq;
#ifndef _win_
extern "C" int fork(void);
@@ -1904,4 +1904,4 @@ IProxyFactory::TPtr CreatePipeFactory(
return new TPipeFactory(options);
}
-} // namespace NYql::NTaskRunnerProxy
+} // namespace NYql::NTaskRunnerProxy
diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.h b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.h
index 137a0e8c0c..635b976db9 100644
--- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.h
+++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_pipe.h
@@ -1,12 +1,12 @@
#pragma once
-#include "tasks_runner_proxy.h"
+#include "tasks_runner_proxy.h"
#include "file_cache.h"
-#include <util/generic/hash.h>
-#include <util/generic/string.h>
+#include <util/generic/hash.h>
+#include <util/generic/string.h>
-namespace NYql::NTaskRunnerProxy {
+namespace NYql::NTaskRunnerProxy {
struct TPipeFactoryOptions {
TString ExecPath;
@@ -21,4 +21,4 @@ struct TPipeFactoryOptions {
IProxyFactory::TPtr CreatePipeFactory(const TPipeFactoryOptions& options);
-} // namespace NYql::NTaskRunnerProxy
+} // namespace NYql::NTaskRunnerProxy
diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.cpp b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.cpp
index 67174b88e9..f307bed4d7 100644
--- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.cpp
+++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.cpp
@@ -1,15 +1,15 @@
#include "tasks_runner_proxy.h"
-namespace NYql::NTaskRunnerProxy {
+namespace NYql::NTaskRunnerProxy {
using namespace NKikimr;
using namespace NDq;
TDqTaskRunnerMemoryLimits DefaultMemoryLimits() {
TDqTaskRunnerMemoryLimits limits;
- limits.ChannelBufferSize = 20_MB;
- limits.OutputChunkMaxSize = 2_MB;
+ limits.ChannelBufferSize = 20_MB;
+ limits.OutputChunkMaxSize = 2_MB;
return limits;
}
-} // namespace NYql::NTaskRunnerProxy
+} // namespace NYql::NTaskRunnerProxy
diff --git a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h
index 31a6e82784..264dc424f5 100644
--- a/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h
+++ b/ydb/library/yql/providers/dq/task_runner/tasks_runner_proxy.h
@@ -5,7 +5,7 @@
#include <ydb/library/yql/minikql/computation/mkql_computation_node_holders.h>
#include <ydb/library/yql/minikql/mkql_node.h>
-namespace NYql::NTaskRunnerProxy {
+namespace NYql::NTaskRunnerProxy {
extern const TString WorkingDirectoryParamName;
extern const TString WorkingDirectoryDontInitParamName; // COMPAT(aozeritsky)
@@ -97,4 +97,4 @@ public:
NDq::TDqTaskRunnerMemoryLimits DefaultMemoryLimits();
-} // namespace NYql::NTaskRunnerProxy
+} // namespace NYql::NTaskRunnerProxy
diff --git a/ydb/library/yql/providers/dq/task_runner/ya.make b/ydb/library/yql/providers/dq/task_runner/ya.make
index ac63d1260d..787f6dd31a 100644
--- a/ydb/library/yql/providers/dq/task_runner/ya.make
+++ b/ydb/library/yql/providers/dq/task_runner/ya.make
@@ -1,8 +1,8 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:yql)
-
-PEERDIR(
+
+PEERDIR(
library/cpp/svnversion
library/cpp/threading/task_scheduler
library/cpp/yson/node
@@ -19,15 +19,15 @@ PEERDIR(
ydb/library/yql/providers/dq/api/protos
ydb/library/yql/providers/dq/counters
ydb/library/yql/providers/dq/interface
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-SRCS(
- file_cache.cpp
- tasks_runner_local.cpp
- tasks_runner_proxy.cpp
- tasks_runner_pipe.cpp
-)
-
-END()
+SRCS(
+ file_cache.cpp
+ tasks_runner_local.cpp
+ tasks_runner_proxy.cpp
+ tasks_runner_pipe.cpp
+)
+
+END()
diff --git a/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp b/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp
index 5ecf0a4a85..278b328d99 100644
--- a/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp
+++ b/ydb/library/yql/providers/dq/worker_manager/local_worker_manager.cpp
@@ -179,8 +179,8 @@ private:
auto& request = ev->Get()->Record.GetRequest();
YQL_ENSURE(request.GetNodeId() == SelfId().NodeId(), "Wrong node id!");
- TFailureInjector::Set(request.GetName(), request.GetSkip(), request.GetCountOfFails());
- YQL_LOG(DEBUG) << "Failure injector is configured " << request.GetName();
+ TFailureInjector::Set(request.GetName(), request.GetSkip(), request.GetCountOfFails());
+ YQL_LOG(DEBUG) << "Failure injector is configured " << request.GetName();
auto response = MakeHolder<TEvConfigureFailureInjectorResponse>();
auto* r = response->Record.MutableResponse();
@@ -209,7 +209,7 @@ private:
YQL_LOG_CTX_SCOPE(ev->Get()->Record.GetTraceId());
YQL_LOG(DEBUG) << "TLocalWorkerManager::TEvAllocateWorkersRequest " << resourceId;
- TFailureInjector::Reach("allocate_workers_failure", [] { ::_exit(1); });
+ TFailureInjector::Reach("allocate_workers_failure", [] { ::_exit(1); });
auto& allocationInfo = AllocatedWorkers[resourceId];
auto traceId = ev->Get()->Record.GetTraceId();
diff --git a/ydb/library/yql/providers/dq/ya.make b/ydb/library/yql/providers/dq/ya.make
index 986edc777d..645f8e956c 100644
--- a/ydb/library/yql/providers/dq/ya.make
+++ b/ydb/library/yql/providers/dq/ya.make
@@ -6,14 +6,14 @@ RECURSE(
config
counters
expr_nodes
- interface
- mkql
+ interface
+ mkql
opt
planner
provider
provider/exec
- runtime
- task_runner
+ runtime
+ task_runner
task_runner_actor
worker_manager
)
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp
index ea67ffd422..7d37279794 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_dq_integration.cpp
@@ -55,7 +55,7 @@ public:
return 0;
}
- ui64 Partition(const TDqSettings&, size_t maxPartitions, const TExprNode& node, TVector<TString>& partitions, TString*, TExprContext&, bool) override {
+ ui64 Partition(const TDqSettings&, size_t maxPartitions, const TExprNode& node, TVector<TString>& partitions, TString*, TExprContext&, bool) override {
if (auto maybePqRead = TMaybeNode<TPqReadTopic>(&node)) {
return PartitionTopicRead(maybePqRead.Cast().Topic(), maxPartitions, partitions);
}
@@ -109,7 +109,7 @@ public:
return read;
}
- TMaybe<bool> CanWrite(const TDqSettings&, const TExprNode&, TExprContext&) override {
+ TMaybe<bool> CanWrite(const TDqSettings&, const TExprNode&, TExprContext&) override {
YQL_ENSURE(false, "Unimplemented");
}
diff --git a/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp b/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp
index 4127452d12..7afe86c4a3 100644
--- a/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp
+++ b/ydb/library/yql/providers/pq/provider/yql_pq_ut.cpp
@@ -110,7 +110,7 @@ bool RunPqProgram(
const auto driverConfig = NYdb::TDriverConfig().SetLog(CreateLogBackend("cerr"));
NYdb::TDriver driver(driverConfig);
- auto dqGateway = CreateLocalDqGateway(driver, functionRegistry.Get(), dqCompFactory, dqTaskTransformFactory, {});
+ auto dqGateway = CreateLocalDqGateway(driver, functionRegistry.Get(), dqCompFactory, dqTaskTransformFactory, {});
auto storage = NYql::CreateFileStorage({});
dataProvidersInit.push_back(NYql::GetDqDataProviderInitializer(&CreateInMemoryExecTransformer, dqGateway, dqCompFactory, {}, storage));
diff --git a/ydb/library/yql/providers/result/expr_nodes/ya.make b/ydb/library/yql/providers/result/expr_nodes/ya.make
index 71181ab3cb..836e32e8e7 100644
--- a/ydb/library/yql/providers/result/expr_nodes/ya.make
+++ b/ydb/library/yql/providers/result/expr_nodes/ya.make
@@ -1,35 +1,35 @@
-LIBRARY()
-
-OWNER(g:yql)
-
-SRCS(
- yql_res_expr_nodes.cpp
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(g:yql)
+
+SRCS(
+ yql_res_expr_nodes.cpp
+)
+
+PEERDIR(
ydb/library/yql/core/expr_nodes
ydb/library/yql/providers/common/provider
-)
-
-SRCDIR(
+)
+
+SRCDIR(
ydb/library/yql/core/expr_nodes_gen
-)
-
-RUN_PROGRAM(
+)
+
+RUN_PROGRAM(
ydb/library/yql/core/expr_nodes_gen/gen
- yql_expr_nodes_gen.jnj
- yql_res_expr_nodes.json
- yql_res_expr_nodes.gen.h
- yql_res_expr_nodes.decl.inl.h
- yql_res_expr_nodes.defs.inl.h
- IN yql_expr_nodes_gen.jnj
- IN yql_res_expr_nodes.json
- OUT yql_res_expr_nodes.gen.h
- OUT yql_res_expr_nodes.decl.inl.h
- OUT yql_res_expr_nodes.defs.inl.h
+ yql_expr_nodes_gen.jnj
+ yql_res_expr_nodes.json
+ yql_res_expr_nodes.gen.h
+ yql_res_expr_nodes.decl.inl.h
+ yql_res_expr_nodes.defs.inl.h
+ IN yql_expr_nodes_gen.jnj
+ IN yql_res_expr_nodes.json
+ OUT yql_res_expr_nodes.gen.h
+ OUT yql_res_expr_nodes.decl.inl.h
+ OUT yql_res_expr_nodes.defs.inl.h
OUTPUT_INCLUDES
${ARCADIA_ROOT}/ydb/library/yql/core/expr_nodes_gen/yql_expr_nodes_gen.h
${ARCADIA_ROOT}/util/generic/hash_set.h
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.cpp b/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.cpp
index 89dd532b0d..e95ec2cb5a 100644
--- a/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.cpp
+++ b/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.cpp
@@ -1,7 +1,7 @@
#include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h>
-
-namespace NYql {
-namespace NNodes {
-
-} // namespace NNodes
-} // namespace NYql
+
+namespace NYql {
+namespace NNodes {
+
+} // namespace NNodes
+} // namespace NYql
diff --git a/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h b/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h
index a9770e379b..0745041aca 100644
--- a/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h
+++ b/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.h
@@ -1,63 +1,63 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.gen.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
-
+
#include <ydb/library/yql/core/expr_nodes/yql_expr_nodes.h>
-
-namespace NYql {
-namespace NNodes {
-
+
+namespace NYql {
+namespace NNodes {
+
#include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.decl.inl.h>
-
-class TResultDataSink: public NGenerated::TResultDataSinkStub<TExprBase, TCallable, TCoAtom> {
-public:
- explicit TResultDataSink(const TExprNode* node)
- : TResultDataSinkStub(node)
- {
- }
-
- explicit TResultDataSink(const TExprNode::TPtr& node)
- : TResultDataSinkStub(node)
- {
- }
-
- static bool Match(const TExprNode* node) {
- if (!TResultDataSinkStub::Match(node)) {
- return false;
- }
-
- if (node->Child(0)->Content() != ResultProviderName) {
- return false;
- }
-
- return true;
- }
-};
-
+
+class TResultDataSink: public NGenerated::TResultDataSinkStub<TExprBase, TCallable, TCoAtom> {
+public:
+ explicit TResultDataSink(const TExprNode* node)
+ : TResultDataSinkStub(node)
+ {
+ }
+
+ explicit TResultDataSink(const TExprNode::TPtr& node)
+ : TResultDataSinkStub(node)
+ {
+ }
+
+ static bool Match(const TExprNode* node) {
+ if (!TResultDataSinkStub::Match(node)) {
+ return false;
+ }
+
+ if (node->Child(0)->Content() != ResultProviderName) {
+ return false;
+ }
+
+ return true;
+ }
+};
+
#include <ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.defs.inl.h>
-
-template<typename TParent>
-class TNodeBuilder<TParent, TResultDataSink> : TNodeBuilderBase
-{
-public:
- typedef std::function<TParent& (const TResultDataSink&)> BuildFuncType;
- typedef std::function<TExprBase (const TStringBuf& arg)> GetArgFuncType;
- typedef TResultDataSink ResultType;
-
+
+template<typename TParent>
+class TNodeBuilder<TParent, TResultDataSink> : TNodeBuilderBase
+{
+public:
+ typedef std::function<TParent& (const TResultDataSink&)> BuildFuncType;
+ typedef std::function<TExprBase (const TStringBuf& arg)> GetArgFuncType;
+ typedef TResultDataSink ResultType;
+
TNodeBuilder(TExprContext& ctx, TPositionHandle pos, BuildFuncType buildFunc, GetArgFuncType getArgFunc)
- : TNodeBuilderBase(ctx, pos, getArgFunc)
- , BuildFunc(buildFunc) {}
-
- TParent& Build() {
- auto atom = this->Ctx.NewAtom(this->Pos, ResultProviderName);
- auto node = this->Ctx.NewCallable(this->Pos, "DataSink", { atom });
- return BuildFunc(TResultDataSink(node));
- }
-
-private:
- BuildFuncType BuildFunc;
-};
-
-} // namespace NNodes
-} // namespace NYql
+ : TNodeBuilderBase(ctx, pos, getArgFunc)
+ , BuildFunc(buildFunc) {}
+
+ TParent& Build() {
+ auto atom = this->Ctx.NewAtom(this->Pos, ResultProviderName);
+ auto node = this->Ctx.NewCallable(this->Pos, "DataSink", { atom });
+ return BuildFunc(TResultDataSink(node));
+ }
+
+private:
+ BuildFuncType BuildFunc;
+};
+
+} // namespace NNodes
+} // namespace NYql
diff --git a/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.json b/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.json
index b7a8ffe180..3911cede27 100644
--- a/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.json
+++ b/ydb/library/yql/providers/result/expr_nodes/yql_res_expr_nodes.json
@@ -1,103 +1,103 @@
-{
- "NodeRootType": "TExprBase",
- "NodeBuilderBase": "TNodeBuilderBase",
- "ListBuilderBase": "TListBuilderBase",
- "FreeArgCallableBase": "TFreeArgCallable",
- "FreeArgBuilderBase": "TFreeArgCallableBuilderBase",
- "Nodes": [
- {
- "Name": "TResultDataSink",
- "Base": "TCallable",
- "Definition": "Custom",
- "Builder": {"Generate": "None"},
- "Match": {"Type": "Callable", "Name": "DataSink"},
- "Children": [
- {"Index": 0, "Name": "Category", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TResBase",
- "Base": "TCallable",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
- "Children": [
- {"Index": 0, "Name": "World", "Type": "TExprBase"},
- {"Index": 1, "Name": "DataSink", "Type": "TResultDataSink"}
- ]
- },
- {
- "Name": "TResWriteBase",
- "Base": "TResBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
- "Children": [
- {"Index": 2, "Name": "Key", "Type": "TCoKey"},
- {"Index": 3, "Name": "Data", "Type": "TExprBase"},
- {"Index": 4, "Name": "Settings", "Type": "TCoNameValueTupleList"}
- ]
- },
- {
- "Name": "TResWrite",
- "Base": "TResWriteBase",
- "Match": {"Type": "Callable", "Name": "ResWrite!"}
- },
- {
- "Name": "TResTransientBase",
- "Base": "TResWriteBase",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
- "Children": [
- {"Index": 5, "Name": "DelegatedSource", "Type": "TCoAtom"}
- ]
- },
- {
- "Name": "TResFill",
- "Base": "TResTransientBase",
- "Match": {"Type": "Callable", "Name": "ResFill!"}
- },
- {
- "Name": "TResPull",
- "Base": "TResTransientBase",
- "Match": {"Type": "Callable", "Name": "ResPull!"}
- },
- {
- "Name": "TResIf",
- "Base": "TResBase",
- "Match": {"Type": "Callable", "Name": "ResIf!"},
- "Children": [
- {"Index": 2, "Name": "Condition", "Type": "TExprBase"},
- {"Index": 3, "Name": "Then", "Type": "TExprBase"},
- {"Index": 4, "Name": "Else", "Type": "TExprBase"},
- {"Index": 5, "Name": "DelegatedSource", "Type": "TCoAtom"},
- {"Index": 6, "Name": "Settings", "Type": "TCoNameValueTupleList"}
- ]
- },
- {
- "Name": "TResOrPullBase",
- "Base": "TCallable",
- "Match": {"Type": "CallableBase"},
- "Builder": {"Generate": "None"},
- "Children": [
- {"Index": 0, "Name": "Input", "Type": "TExprBase"},
- {"Index": 1, "Name": "BytesLimit", "Type": "TCoAtom"},
- {"Index": 2, "Name": "RowsLimit", "Type": "TCoAtom"},
- {"Index": 3, "Name": "FormatDetails", "Type": "TCoAtom"},
- {"Index": 4, "Name": "Settings", "Type": "TCoNameValueTupleList"},
- {"Index": 5, "Name": "Format", "Type": "TCoAtom"},
+{
+ "NodeRootType": "TExprBase",
+ "NodeBuilderBase": "TNodeBuilderBase",
+ "ListBuilderBase": "TListBuilderBase",
+ "FreeArgCallableBase": "TFreeArgCallable",
+ "FreeArgBuilderBase": "TFreeArgCallableBuilderBase",
+ "Nodes": [
+ {
+ "Name": "TResultDataSink",
+ "Base": "TCallable",
+ "Definition": "Custom",
+ "Builder": {"Generate": "None"},
+ "Match": {"Type": "Callable", "Name": "DataSink"},
+ "Children": [
+ {"Index": 0, "Name": "Category", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TResBase",
+ "Base": "TCallable",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
+ "Children": [
+ {"Index": 0, "Name": "World", "Type": "TExprBase"},
+ {"Index": 1, "Name": "DataSink", "Type": "TResultDataSink"}
+ ]
+ },
+ {
+ "Name": "TResWriteBase",
+ "Base": "TResBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
+ "Children": [
+ {"Index": 2, "Name": "Key", "Type": "TCoKey"},
+ {"Index": 3, "Name": "Data", "Type": "TExprBase"},
+ {"Index": 4, "Name": "Settings", "Type": "TCoNameValueTupleList"}
+ ]
+ },
+ {
+ "Name": "TResWrite",
+ "Base": "TResWriteBase",
+ "Match": {"Type": "Callable", "Name": "ResWrite!"}
+ },
+ {
+ "Name": "TResTransientBase",
+ "Base": "TResWriteBase",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
+ "Children": [
+ {"Index": 5, "Name": "DelegatedSource", "Type": "TCoAtom"}
+ ]
+ },
+ {
+ "Name": "TResFill",
+ "Base": "TResTransientBase",
+ "Match": {"Type": "Callable", "Name": "ResFill!"}
+ },
+ {
+ "Name": "TResPull",
+ "Base": "TResTransientBase",
+ "Match": {"Type": "Callable", "Name": "ResPull!"}
+ },
+ {
+ "Name": "TResIf",
+ "Base": "TResBase",
+ "Match": {"Type": "Callable", "Name": "ResIf!"},
+ "Children": [
+ {"Index": 2, "Name": "Condition", "Type": "TExprBase"},
+ {"Index": 3, "Name": "Then", "Type": "TExprBase"},
+ {"Index": 4, "Name": "Else", "Type": "TExprBase"},
+ {"Index": 5, "Name": "DelegatedSource", "Type": "TCoAtom"},
+ {"Index": 6, "Name": "Settings", "Type": "TCoNameValueTupleList"}
+ ]
+ },
+ {
+ "Name": "TResOrPullBase",
+ "Base": "TCallable",
+ "Match": {"Type": "CallableBase"},
+ "Builder": {"Generate": "None"},
+ "Children": [
+ {"Index": 0, "Name": "Input", "Type": "TExprBase"},
+ {"Index": 1, "Name": "BytesLimit", "Type": "TCoAtom"},
+ {"Index": 2, "Name": "RowsLimit", "Type": "TCoAtom"},
+ {"Index": 3, "Name": "FormatDetails", "Type": "TCoAtom"},
+ {"Index": 4, "Name": "Settings", "Type": "TCoNameValueTupleList"},
+ {"Index": 5, "Name": "Format", "Type": "TCoAtom"},
{"Index": 6, "Name": "PublicId", "Type": "TCoAtom"},
- {"Index": 7, "Name": "Discard", "Type": "TCoAtom"},
- {"Index": 8, "Name": "Origin", "Type": "TExprBase"}
- ]
- },
- {
- "Name": "TResult",
- "Base": "TResOrPullBase",
- "Match": {"Type": "Callable", "Name": "Result"}
- },
- {
- "Name": "TPull",
- "Base": "TResOrPullBase",
- "Match": {"Type": "Callable", "Name": "Pull"}
+ {"Index": 7, "Name": "Discard", "Type": "TCoAtom"},
+ {"Index": 8, "Name": "Origin", "Type": "TExprBase"}
+ ]
+ },
+ {
+ "Name": "TResult",
+ "Base": "TResOrPullBase",
+ "Match": {"Type": "Callable", "Name": "Result"}
+ },
+ {
+ "Name": "TPull",
+ "Base": "TResOrPullBase",
+ "Match": {"Type": "Callable", "Name": "Pull"}
},
{
"Name": "TResFor",
@@ -113,6 +113,6 @@
{"Index": 8, "Name": "Current", "Type": "TCoAtom"},
{"Index": 9, "Name": "Active", "Type": "TExprBase"}
]
- }
- ]
-}
+ }
+ ]
+}
diff --git a/ydb/library/yql/providers/result/provider/ya.make b/ydb/library/yql/providers/result/provider/ya.make
index 66915b115f..129c19cb58 100644
--- a/ydb/library/yql/providers/result/provider/ya.make
+++ b/ydb/library/yql/providers/result/provider/ya.make
@@ -1,13 +1,13 @@
-LIBRARY()
-
-OWNER(g:yql)
-
-SRCS(
- yql_result_provider.cpp
- yql_result_provider.h
-)
-
-PEERDIR(
+LIBRARY()
+
+OWNER(g:yql)
+
+SRCS(
+ yql_result_provider.cpp
+ yql_result_provider.h
+)
+
+PEERDIR(
library/cpp/yson/node
ydb/library/yql/ast
ydb/library/yql/utils/log
@@ -17,8 +17,8 @@ PEERDIR(
ydb/library/yql/providers/common/mkql
ydb/library/yql/providers/common/provider
ydb/library/yql/providers/result/expr_nodes
-)
-
+)
+
YQL_LAST_ABI_VERSION()
-END()
+END()
diff --git a/ydb/library/yql/providers/result/provider/yql_result_provider.cpp b/ydb/library/yql/providers/result/provider/yql_result_provider.cpp
index 13bfd5e0f7..4ab051a016 100644
--- a/ydb/library/yql/providers/result/provider/yql_result_provider.cpp
+++ b/ydb/library/yql/providers/result/provider/yql_result_provider.cpp
@@ -97,7 +97,7 @@ namespace {
IGraphTransformer::TStatus ValidateColumns(TExprNode::TPtr& columns, const TTypeAnnotationNode* listType, TExprContext& ctx) {
bool hasPrefixes = false;
for (auto& child : columns->Children()) {
- if (HasError(child->GetTypeAnn(), ctx)) {
+ if (HasError(child->GetTypeAnn(), ctx)) {
return IGraphTransformer::TStatus::Error;
}
@@ -202,20 +202,20 @@ namespace {
TStatus DoTransform(TExprNode::TPtr input, TExprNode::TPtr& output, TExprContext& ctx) final {
output = input;
-
- TString uniqId = TStringBuilder() << '#' << input->UniqueId();
- YQL_LOG_CTX_SCOPE(uniqId);
-
+
+ TString uniqId = TStringBuilder() << '#' << input->UniqueId();
+ YQL_LOG_CTX_SCOPE(uniqId);
+
YQL_ENSURE(input->Type() == TExprNode::Callable);
- TExprBase node(input);
+ TExprBase node(input);
if (node.Maybe<TResFill>() || node.Maybe<TResPull>() || node.Maybe<TResIf>() || node.Maybe<TResFor>()) {
- auto provider = Config->Types.DataSourceMap.FindPtr(input->Child(5)->Content());
+ auto provider = Config->Types.DataSourceMap.FindPtr(input->Child(5)->Content());
Y_ENSURE(provider, "DataSource not exist: " << input->Child(5)->Content());
- if (node.Maybe<TResPull>()) {
+ if (node.Maybe<TResPull>()) {
return HandleFillOrPull<TPull>(node, output, ctx, *(*provider));
- } else {
+ } else {
return HandleFillOrPull<TResult>(node, output, ctx, *(*provider));
- }
+ }
}
if (input->Content() == CommitName) {
@@ -268,43 +268,43 @@ namespace {
}
private:
- template <class TTarget>
+ template <class TTarget>
bool& GetOverflowFlagAndCommitedSize(ui64& committed);
template <class TTarget>
TStatus HandleFillOrPull(TExprBase input, TExprNode::TPtr& output, TExprContext& ctx, IDataProvider& provider) {
- auto requireWorld = RequireChild(input.Ref(), TResBase::idx_World);
- auto requireData = input.Maybe<TResPull>() ? RequireChild(input.Ref(), TResPull::idx_Data) : IGraphTransformer::TStatus::Ok;
+ auto requireWorld = RequireChild(input.Ref(), TResBase::idx_World);
+ auto requireData = input.Maybe<TResPull>() ? RequireChild(input.Ref(), TResPull::idx_Data) : IGraphTransformer::TStatus::Ok;
auto requireStatus = requireWorld.Combine(requireData);
if (requireStatus.Level != IGraphTransformer::TStatus::Ok) {
return requireStatus;
}
bool needWriter = true;
- TMaybeNode<TExprBase> dataNode;
- TMaybeNode<TCoNameValueTupleList> options;
- if (input.Maybe<TResIf>()) {
- if (input.Ref().HasResult()) {
- auto resultYsonString = input.Ref().GetResult().Content();
+ TMaybeNode<TExprBase> dataNode;
+ TMaybeNode<TCoNameValueTupleList> options;
+ if (input.Maybe<TResIf>()) {
+ if (input.Ref().HasResult()) {
+ auto resultYsonString = input.Ref().GetResult().Content();
auto resultNode = NYT::NodeFromYsonString(TString(resultYsonString), ::NYson::EYsonType::Node);
YQL_ENSURE(resultNode.IsMap());
auto resultBoolNode = resultNode.AsMap()["Data"];
YQL_ENSURE(resultBoolNode.IsBool());
const bool predicate = resultBoolNode.AsBool();
- auto branchStatus = RequireChild(input.Ref(), predicate ? TResIf::idx_Then : TResIf::idx_Else);
+ auto branchStatus = RequireChild(input.Ref(), predicate ? TResIf::idx_Then : TResIf::idx_Else);
if (branchStatus.Level != IGraphTransformer::TStatus::Ok) {
return branchStatus;
}
- input.Ptr()->SetResult(ctx.NewWorld(input.Pos()));
+ input.Ptr()->SetResult(ctx.NewWorld(input.Pos()));
input.Ptr()->SetState(TExprNode::EState::ExecutionComplete);
return TStatus::Ok;
}
needWriter = false;
- dataNode = input.Cast<TResIf>().Condition();
- options = input.Cast<TResIf>().Settings();
+ dataNode = input.Cast<TResIf>().Condition();
+ options = input.Cast<TResIf>().Settings();
} else if (input.Maybe<TResFor>()) {
const auto& forNode = input.Cast<TResFor>();
if (forNode.Total().Value()) {
@@ -457,9 +457,9 @@ namespace {
needWriter = false;
dataNode = forNode.Items();
options = forNode.Settings();
- } else {
- dataNode = input.Cast<TResWriteBase>().Data();
- options = input.Cast<TResWriteBase>().Settings();
+ } else {
+ dataNode = input.Cast<TResWriteBase>().Data();
+ options = input.Cast<TResWriteBase>().Settings();
}
DelegatedProvider = &provider;
@@ -485,9 +485,9 @@ namespace {
auto rowsLimit = fillSettings.RowsLimitPerWrite;
bool discard = false;
TString label;
- for (auto setting : options.Cast()) {
- if (setting.Name().Value() == "take") {
- auto value = FromString<ui64>(setting.Value().Cast<TCoAtom>().Value());
+ for (auto setting : options.Cast()) {
+ if (setting.Name().Value() == "take") {
+ auto value = FromString<ui64>(setting.Value().Cast<TCoAtom>().Value());
if (rowsLimit) {
rowsLimit = Min(*rowsLimit, value);
} else {
@@ -501,10 +501,10 @@ namespace {
}
TString publicId;
- if (auto id = Config->Types.TranslateOperationId(input.Ref().UniqueId())) {
- publicId = ToString(*id);
+ if (auto id = Config->Types.TranslateOperationId(input.Ref().UniqueId())) {
+ publicId = ToString(*id);
}
-
+
if (needWriter && !ResultWriter) {
YQL_ENSURE(Config->WriterFactory);
ResultWriter = Config->WriterFactory();
@@ -522,42 +522,42 @@ namespace {
rowsLimit.Clear();
}
- DelegatedNode = Build<TTarget>(ctx, input.Pos())
- .Input(dataNode.Cast())
- .BytesLimit()
- .Value(fillSettings.AllResultsBytesLimit ? ToString(*fillSettings.AllResultsBytesLimit) : TString())
- .Build()
- .RowsLimit()
- .Value(rowsLimit ? ToString(*rowsLimit) : TString())
- .Build()
- .FormatDetails()
- .Value(fillSettings.FormatDetails)
- .Build()
- .Settings(options.Cast())
- .Format()
- .Value(ToString((ui32)fillSettings.Format))
- .Build()
- .PublicId()
- .Value(publicId)
- .Build()
+ DelegatedNode = Build<TTarget>(ctx, input.Pos())
+ .Input(dataNode.Cast())
+ .BytesLimit()
+ .Value(fillSettings.AllResultsBytesLimit ? ToString(*fillSettings.AllResultsBytesLimit) : TString())
+ .Build()
+ .RowsLimit()
+ .Value(rowsLimit ? ToString(*rowsLimit) : TString())
+ .Build()
+ .FormatDetails()
+ .Value(fillSettings.FormatDetails)
+ .Build()
+ .Settings(options.Cast())
+ .Format()
+ .Value(ToString((ui32)fillSettings.Format))
+ .Build()
+ .PublicId()
+ .Value(publicId)
+ .Build()
.Discard()
.Value(ToString(discard))
.Build()
- .Origin(input)
- .Done().Ptr();
-
+ .Origin(input)
+ .Done().Ptr();
+
for (auto idx: {TResOrPullBase::idx_BytesLimit, TResOrPullBase::idx_RowsLimit, TResOrPullBase::idx_FormatDetails,
TResOrPullBase::idx_Format, TResOrPullBase::idx_PublicId, TResOrPullBase::idx_Discard }) {
- DelegatedNode->Child(idx)->SetTypeAnn(atomType);
+ DelegatedNode->Child(idx)->SetTypeAnn(atomType);
DelegatedNode->Child(idx)->SetState(TExprNode::EState::ConstrComplete);
- }
-
+ }
+
DelegatedNode->SetTypeAnn(input.Ref().GetTypeAnn());
DelegatedNode->SetState(TExprNode::EState::ConstrComplete);
input.Ptr()->SetState(TExprNode::EState::ExecutionInProgress);
auto status = DelegatedProvider->GetCallableExecutionTransformer().Transform(DelegatedNode, DelegatedNodeOutput, ctx);
if (status.Level != TStatus::Async) {
- FinishNode(*input.Ptr(), ctx, status);
+ FinishNode(*input.Ptr(), ctx, status);
}
return status;
@@ -674,7 +674,7 @@ namespace {
return nullptr;
}
- auto fillSettings = Config->FillSettings;
+ auto fillSettings = Config->FillSettings;
if (!isRef && (!isAutoRef || !canRef)) {
for (auto setting: resWrite.Settings()) {
if (setting.Name().Value() == "take") {
@@ -690,10 +690,10 @@ namespace {
if (fillSettings.RowsLimitPerWrite) {
*fillSettings.RowsLimitPerWrite += 1;
}
- } else {
- fillSettings.RowsLimitPerWrite.Clear();
+ } else {
+ fillSettings.RowsLimitPerWrite.Clear();
}
- newInput = TExprBase(source->OptimizePull(newInput.Ptr(), fillSettings, ctx, optCtx));
+ newInput = TExprBase(source->OptimizePull(newInput.Ptr(), fillSettings, ctx, optCtx));
ret = Build<TResPull>(ctx, resWrite.Pos())
.World(ApplySyncListToWorld(resWrite.World().Ptr(), syncList, ctx))
@@ -859,16 +859,16 @@ namespace {
TProcessedNodesSet PhysicalOptProcessedNodes;
};
- class TResultProvider : public TDataProviderBase {
+ class TResultProvider : public TDataProviderBase {
public:
struct TFunctions {
THashSet<TStringBuf> Names;
TFunctions() {
- Names.insert(TResWrite::CallableName());
- Names.insert(TResFill::CallableName());
- Names.insert(TResPull::CallableName());
- Names.insert(TResIf::CallableName());
+ Names.insert(TResWrite::CallableName());
+ Names.insert(TResFill::CallableName());
+ Names.insert(TResPull::CallableName());
+ Names.insert(TResIf::CallableName());
Names.insert(TResFor::CallableName());
}
};
@@ -882,7 +882,7 @@ namespace {
return ResultProviderName;
}
- bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
+ bool ValidateParameters(TExprNode& node, TExprContext& ctx, TMaybe<TString>& cluster) override {
if (!EnsureArgsCount(node, 1, ctx)) {
return false;
}
@@ -895,41 +895,41 @@ namespace {
return ResultProviderFunctions().contains(node.Content()) || node.Content() == ConfigureName;
}
- void FillModifyCallables(THashSet<TStringBuf>& callables) override {
- callables.insert(TResWrite::CallableName());
- }
-
+ void FillModifyCallables(THashSet<TStringBuf>& callables) override {
+ callables.insert(TResWrite::CallableName());
+ }
+
IGraphTransformer& GetTypeAnnotationTransformer(bool instantOnly) override {
Y_UNUSED(instantOnly);
if (!TypeAnnotationTransformer) {
TypeAnnotationTransformer = CreateFunctorTransformer(
[&](const TExprNode::TPtr& input, TExprNode::TPtr& output, TExprContext& ctx)->IGraphTransformer::TStatus {
output = input;
-
- if (auto maybeRes = TMaybeNode<TResWriteBase>(input)) {
- auto res = maybeRes.Cast();
- if (!EnsureArgsCount(*input, res.Maybe<TResWrite>() ? 5 : 6, ctx)) {
+
+ if (auto maybeRes = TMaybeNode<TResWriteBase>(input)) {
+ auto res = maybeRes.Cast();
+ if (!EnsureArgsCount(*input, res.Maybe<TResWrite>() ? 5 : 6, ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!EnsureWorldType(*res.Ref().Child(TResWriteBase::idx_World), ctx)) {
+ if (!EnsureWorldType(*res.Ref().Child(TResWriteBase::idx_World), ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!EnsureSpecificDataSink(*res.Ref().Child(TResWriteBase::idx_DataSink), ResultProviderName, ctx)) {
+ if (!EnsureSpecificDataSink(*res.Ref().Child(TResWriteBase::idx_DataSink), ResultProviderName, ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!res.Ref().Child(TResWriteBase::idx_Key)->IsCallable("Key") || res.Ref().Child(TResWriteBase::idx_Key)->ChildrenSize() > 0) {
+ if (!res.Ref().Child(TResWriteBase::idx_Key)->IsCallable("Key") || res.Ref().Child(TResWriteBase::idx_Key)->ChildrenSize() > 0) {
ctx.AddError(TIssue(ctx.GetPosition(res.Ref().Child(TResWriteBase::idx_Key)->Pos()), "Expected empty key"));
return IGraphTransformer::TStatus::Error;
}
- if (!EnsurePersistable(res.Data().Ref(), ctx)) {
+ if (!EnsurePersistable(res.Data().Ref(), ctx)) {
return IGraphTransformer::TStatus::Error;
}
- auto settings = res.Ref().Child(TResWriteBase::idx_Settings);
+ auto settings = res.Ref().Child(TResWriteBase::idx_Settings);
if (!EnsureTuple(*settings, ctx)) {
return IGraphTransformer::TStatus::Error;
}
@@ -987,7 +987,7 @@ namespace {
}
columns = setting->ChildPtr(1);
- auto status = ValidateColumns(columns, res.Data().Ref().GetTypeAnn(), ctx);
+ auto status = ValidateColumns(columns, res.Data().Ref().GetTypeAnn(), ctx);
if (status.Level != IGraphTransformer::TStatus::Ok) {
if (status.Level == IGraphTransformer::TStatus::Repeat) {
auto newSetting = ctx.ChangeChild(*setting, 1, std::move(columns));
@@ -1026,20 +1026,20 @@ namespace {
++settingPos;
}
- if (auto right = res.Data().Maybe<TCoRight>()) {
- auto source = right.Cast().Input();
+ if (auto right = res.Data().Maybe<TCoRight>()) {
+ auto source = right.Cast().Input();
const TIntrusivePtr<IDataProvider>* provider = nullptr;
- if (source.Ref().Type() == TExprNode::Callable || source.Ref().ChildrenSize() >= 2) {
+ if (source.Ref().Type() == TExprNode::Callable || source.Ref().ChildrenSize() >= 2) {
if (source.Ref().Child(1)->IsCallable("DataSource")) {
auto name = source.Ref().Child(1)->Child(0)->Content();
- provider = Config->Types.DataSourceMap.FindPtr(name);
+ provider = Config->Types.DataSourceMap.FindPtr(name);
Y_ENSURE(provider, "DataSource doesn't exist: " << name);
}
if (source.Ref().Child(1)->IsCallable("DataSink")) {
auto name = source.Ref().Child(1)->Child(0)->Content();
- provider = Config->Types.DataSinkMap.FindPtr(name);
+ provider = Config->Types.DataSinkMap.FindPtr(name);
Y_ENSURE(provider, "DataSink doesn't exist: " << name);
}
}
@@ -1050,15 +1050,15 @@ namespace {
}
}
- if (res.Maybe<TResTransientBase>()) {
- auto resTransient = res.Cast<TResTransientBase>();
- if (!EnsureAtom(*resTransient.Ref().Child(TResTransientBase::idx_DelegatedSource), ctx)) {
+ if (res.Maybe<TResTransientBase>()) {
+ auto resTransient = res.Cast<TResTransientBase>();
+ if (!EnsureAtom(*resTransient.Ref().Child(TResTransientBase::idx_DelegatedSource), ctx)) {
return IGraphTransformer::TStatus::Error;
}
- if (!Config->Types.DataSourceMap.FindPtr(resTransient.DelegatedSource().Value())) {
+ if (!Config->Types.DataSourceMap.FindPtr(resTransient.DelegatedSource().Value())) {
ctx.AddError(TIssue(ctx.GetPosition(resTransient.DelegatedSource().Pos()),
- TStringBuilder() << "DataSource is not found: " << resTransient.DelegatedSource().Value()));
+ TStringBuilder() << "DataSource is not found: " << resTransient.DelegatedSource().Value()));
return IGraphTransformer::TStatus::Error;
}
}
@@ -1082,10 +1082,10 @@ namespace {
return IGraphTransformer::TStatus::Repeat;
}
- input->SetTypeAnn(res.World().Ref().GetTypeAnn());
+ input->SetTypeAnn(res.World().Ref().GetTypeAnn());
return IGraphTransformer::TStatus::Ok;
}
- else if (auto maybeIf = TMaybeNode<TResIf>(input)) {
+ else if (auto maybeIf = TMaybeNode<TResIf>(input)) {
if (!EnsureArgsCount(*input, 7, ctx)) {
return IGraphTransformer::TStatus::Error;
}
@@ -1242,7 +1242,7 @@ namespace {
input->SetTypeAnn(input->Child(TResIf::idx_World)->GetTypeAnn());
return IGraphTransformer::TStatus::Ok;
}
- else if (auto maybeCommit = TMaybeNode<TCoCommit>(input)) {
+ else if (auto maybeCommit = TMaybeNode<TCoCommit>(input)) {
auto commit = maybeCommit.Cast();
auto settings = NCommon::ParseCommitSettings(commit, ctx);
@@ -1256,7 +1256,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(commit.World().Ref().GetTypeAnn());
+ input->SetTypeAnn(commit.World().Ref().GetTypeAnn());
return IGraphTransformer::TStatus::Ok;
}
else if (input->Content() == ConfigureName) {
@@ -1304,7 +1304,7 @@ namespace {
return IGraphTransformer::TStatus::Error;
}
- input->SetTypeAnn(input->Child(0)->GetTypeAnn());
+ input->SetTypeAnn(input->Child(0)->GetTypeAnn());
return IGraphTransformer::TStatus::Ok;
}
@@ -1317,19 +1317,19 @@ namespace {
}
TExprNode::TPtr RewriteIO(const TExprNode::TPtr& node, TExprContext& ctx) override {
- auto ret = node;
+ auto ret = node;
if (node->Content() == WriteName) {
- ret = ctx.RenameNode(*ret, TResWrite::CallableName());
- ret = ctx.ChangeChild(*ret, TResWrite::idx_Data,
- ctx.Builder(node->Pos())
- .Callable("RemovePrefixMembers")
- .Add(0, node->ChildPtr(TResWrite::idx_Data))
- .List(1)
- .Atom(0, "_yql_sys_", TNodeFlags::Default)
- .Seal()
- .Seal()
- .Build()
- );
+ ret = ctx.RenameNode(*ret, TResWrite::CallableName());
+ ret = ctx.ChangeChild(*ret, TResWrite::idx_Data,
+ ctx.Builder(node->Pos())
+ .Callable("RemovePrefixMembers")
+ .Add(0, node->ChildPtr(TResWrite::idx_Data))
+ .List(1)
+ .Atom(0, "_yql_sys_", TNodeFlags::Default)
+ .Seal()
+ .Seal()
+ .Build()
+ );
}
else {
YQL_ENSURE(false, "Expected Write!");
@@ -1348,15 +1348,15 @@ namespace {
}
bool CanExecute(const TExprNode& node) override {
- if (node.Content() == TResFill::CallableName()) {
+ if (node.Content() == TResFill::CallableName()) {
return true;
}
- if (node.Content() == TResPull::CallableName()) {
+ if (node.Content() == TResPull::CallableName()) {
return true;
}
- if (node.Content() == TResIf::CallableName()) {
+ if (node.Content() == TResIf::CallableName()) {
return true;
}
@@ -1390,20 +1390,20 @@ namespace {
bool GetDependencies(const TExprNode& node, TExprNode::TListType& children, bool compact) override {
if (CanExecute(node)) {
children.push_back(node.ChildPtr(0));
- if (auto resPull = TMaybeNode<TResPull>(&node)) {
- children.push_back(resPull.Cast().Data().Ptr());
- } else if (auto resIf = TMaybeNode<TResIf>(&node)) {
- children.push_back(resIf.Cast().Then().Ptr());
- children.push_back(resIf.Cast().Else().Ptr());
+ if (auto resPull = TMaybeNode<TResPull>(&node)) {
+ children.push_back(resPull.Cast().Data().Ptr());
+ } else if (auto resIf = TMaybeNode<TResIf>(&node)) {
+ children.push_back(resIf.Cast().Then().Ptr());
+ children.push_back(resIf.Cast().Else().Ptr());
} else if (auto resFor = TMaybeNode<TResFor>(&node)) {
auto active = resFor.Cast().Active().Ptr();
if (!active->IsCallable("Void")) {
children.push_back(active);
}
- } else if (auto resFill = TMaybeNode<TResFill>(&node)) {
- const auto provider = Config->Types.DataSourceMap.FindPtr(resFill.Cast().DelegatedSource().Value());
- Y_ENSURE(provider, "DataSource not exist: " << resFill.Cast().DelegatedSource().Value());
- (*provider)->GetPlanFormatter().GetResultDependencies(resFill.Cast().Data().Ptr(), children, compact);
+ } else if (auto resFill = TMaybeNode<TResFill>(&node)) {
+ const auto provider = Config->Types.DataSourceMap.FindPtr(resFill.Cast().DelegatedSource().Value());
+ Y_ENSURE(provider, "DataSource not exist: " << resFill.Cast().DelegatedSource().Value());
+ (*provider)->GetPlanFormatter().GetResultDependencies(resFill.Cast().Data().Ptr(), children, compact);
}
return true;
@@ -1413,12 +1413,12 @@ namespace {
}
void WritePlanDetails(const TExprNode& node, NYson::TYsonWriter& writer) override {
- if (auto resPull = TMaybeNode<TResPull>(&node)) {
- auto dataSourceName = resPull.Cast().DelegatedSource().Value();
- auto dataSource = Config->Types.DataSourceMap.FindPtr(dataSourceName);
+ if (auto resPull = TMaybeNode<TResPull>(&node)) {
+ auto dataSourceName = resPull.Cast().DelegatedSource().Value();
+ auto dataSource = Config->Types.DataSourceMap.FindPtr(dataSourceName);
YQL_ENSURE(dataSource);
- (*dataSource)->GetPlanFormatter().WritePullDetails(resPull.Cast().Data().Ref(), writer);
+ (*dataSource)->GetPlanFormatter().WritePullDetails(resPull.Cast().Data().Ref(), writer);
}
}
@@ -1428,7 +1428,7 @@ namespace {
}
TString GetOperationDisplayName(const TExprNode& node) override {
- if (node.Content() == CommitName) {
+ if (node.Content() == CommitName) {
return TString::Join(node.Content(), " on result");
}
diff --git a/ydb/library/yql/providers/result/provider/yql_result_provider.h b/ydb/library/yql/providers/result/provider/yql_result_provider.h
index ee5bb5a9f6..b89078be2e 100644
--- a/ydb/library/yql/providers/result/provider/yql_result_provider.h
+++ b/ydb/library/yql/providers/result/provider/yql_result_provider.h
@@ -26,7 +26,7 @@ struct TResultProviderConfig : TThrRefBase {
TResultProviderConfig(TTypeAnnotationContext& types,
const NKikimr::NMiniKQL::IFunctionRegistry& functionRegistry,
- IDataProvider::EResultFormat format, const TString& formatDetails, TResultWriterFactory writerFactory)
+ IDataProvider::EResultFormat format, const TString& formatDetails, TResultWriterFactory writerFactory)
: Types(types)
, FunctionRegistry(functionRegistry)
, WriterFactory(writerFactory)
diff --git a/ydb/library/yql/providers/result/ya.make b/ydb/library/yql/providers/result/ya.make
index 14a0d3a736..c8d5d269f0 100644
--- a/ydb/library/yql/providers/result/ya.make
+++ b/ydb/library/yql/providers/result/ya.make
@@ -1,4 +1,4 @@
-RECURSE(
- expr_nodes
- provider
+RECURSE(
+ expr_nodes
+ provider
)
diff --git a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp
index 539272add6..c4105e945b 100644
--- a/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp
+++ b/ydb/library/yql/providers/s3/provider/yql_s3_dq_integration.cpp
@@ -23,7 +23,7 @@ public:
{
}
- ui64 Partition(const TDqSettings&, size_t maxPartitions, const TExprNode& node, TVector<TString>& partitions, TString*, TExprContext&, bool) override {
+ ui64 Partition(const TDqSettings&, size_t maxPartitions, const TExprNode& node, TVector<TString>& partitions, TString*, TExprContext&, bool) override {
TString cluster;
std::vector<std::vector<TString>> parts;
if (const TMaybeNode<TDqSource> source = &node) {
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp
index b09fd381a0..cea38d2a45 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_datasink.cpp
@@ -64,10 +64,10 @@ public:
return TypeAnnotationTransformer_->CanParse(node);
}
- void FillModifyCallables(THashSet<TStringBuf>& callables) override {
- callables.insert(TSoWriteToShard::CallableName());
- }
-
+ void FillModifyCallables(THashSet<TStringBuf>& callables) override {
+ callables.insert(TSoWriteToShard::CallableName());
+ }
+
bool CanExecute(const TExprNode& node) override {
return ExecutionTransformer_->CanExec(node);
}
diff --git a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp
index 773c960991..a1ca07091a 100644
--- a/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp
+++ b/ydb/library/yql/providers/solomon/provider/yql_solomon_dq_integration.cpp
@@ -98,7 +98,7 @@ public:
YQL_ENSURE(false, "Unimplemented");
}
- TMaybe<bool> CanWrite(const TDqSettings&, const TExprNode&, TExprContext&) override {
+ TMaybe<bool> CanWrite(const TDqSettings&, const TExprNode&, TExprContext&) override {
YQL_ENSURE(false, "Unimplemented");
}
diff --git a/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp b/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp
index ecb85e824b..aaeb081f7d 100644
--- a/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp
+++ b/ydb/library/yql/providers/ydb/provider/yql_ydb_dq_integration.cpp
@@ -23,8 +23,8 @@ public:
{
}
- ui64 Partition(const TDqSettings& settings, size_t maxPartitions, const TExprNode& node,
- TVector<TString>& partitions, TString*, TExprContext&, bool) override {
+ ui64 Partition(const TDqSettings& settings, size_t maxPartitions, const TExprNode& node,
+ TVector<TString>& partitions, TString*, TExprContext&, bool) override {
TString cluster, table;
if (const TMaybeNode<TDqSource> source = &node) {
cluster = source.Cast().DataSource().Cast<TYdbDataSource>().Cluster().Value();
diff --git a/ydb/library/yql/public/decimal/ya.make b/ydb/library/yql/public/decimal/ya.make
index d3c8fbe4e6..2656cc5e10 100644
--- a/ydb/library/yql/public/decimal/ya.make
+++ b/ydb/library/yql/public/decimal/ya.make
@@ -11,8 +11,8 @@ OWNER(
SRCS(
yql_decimal.h
yql_decimal.cpp
- yql_decimal_serialize.h
- yql_decimal_serialize.cpp
+ yql_decimal_serialize.h
+ yql_decimal_serialize.cpp
)
END()
diff --git a/ydb/library/yql/public/decimal/yql_decimal.cpp b/ydb/library/yql/public/decimal/yql_decimal.cpp
index 89fc92cd49..9cac21f038 100644
--- a/ydb/library/yql/public/decimal/yql_decimal.cpp
+++ b/ydb/library/yql/public/decimal/yql_decimal.cpp
@@ -121,13 +121,13 @@ namespace {
TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) {
if (scale > precision)
- return Err();
+ return Err();
auto s = str.data();
auto l = str.size();
if (!s || !l)
- return Err();
+ return Err();
const bool neg = '-' == *s;
if (neg || '+' == *s) {
@@ -148,7 +148,7 @@ TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) {
for (bool dot = false; l; --l) {
if (*s == '.') {
if (dot)
- return Err();
+ return Err();
++s;
dot = true;
@@ -164,7 +164,7 @@ TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) {
const char c = *s++;
if (!std::isdigit(c))
- return Err();
+ return Err();
v *= Ten;
v += c - '0';
@@ -177,14 +177,14 @@ TInt128 FromString(const TStringBuf& str, ui8 precision, ui8 scale) {
if (l--) {
const char c = *s++;
if (!std::isdigit(c))
- return Err();
+ return Err();
bool plus = c > '5';
if (!plus && c == '5') {
for (plus = v & 1; !plus && l; --l) {
const char c = *s++;
if (!std::isdigit(c))
- return Err();
+ return Err();
plus = c != '0';
}
diff --git a/ydb/library/yql/public/decimal/yql_decimal.h b/ydb/library/yql/public/decimal/yql_decimal.h
index 20695f0d84..3c1a57a848 100644
--- a/ydb/library/yql/public/decimal/yql_decimal.h
+++ b/ydb/library/yql/public/decimal/yql_decimal.h
@@ -3,9 +3,9 @@
#include <util/generic/strbuf.h>
#include "yql_wide_int.h"
-#include <type_traits>
-#include <limits>
-
+#include <type_traits>
+#include <limits>
+
namespace NYql {
namespace NDecimal {
@@ -44,10 +44,10 @@ inline constexpr TInt128 Nan() {
return Inf() + TInt128(1);
}
-inline constexpr TInt128 Err() {
- return Nan() + TInt128(1);
-}
-
+inline constexpr TInt128 Err() {
+ return Nan() + TInt128(1);
+}
+
TUint128 GetDivider(ui8 scale);
template<ui8 Precision>
@@ -87,48 +87,48 @@ inline TInt128 FromProto(const TMkqlProto& val) {
return val128;
}
-template<typename TValue>
-inline constexpr TValue YtDecimalNan() {
- return std::numeric_limits<TValue>::max();
-}
-
-template<>
-inline constexpr TInt128 YtDecimalNan<TInt128>() {
- return ~(TInt128(1) << 127);
-}
-
-template<typename TValue>
-inline constexpr TValue YtDecimalInf() {
- return YtDecimalNan<TValue>() - 1;
-}
-
-template<typename TValue>
-inline TInt128 FromYtDecimal(TValue val) {
- static_assert(std::is_same<TInt128, TValue>::value || std::is_signed<TValue>::value, "Expected signed value");
- if (YtDecimalNan<TValue>() == val) {
- return Nan();
- } else if (YtDecimalInf<TValue>() == val) {
- return Inf();
- } else if (-YtDecimalInf<TValue>() == val) {
- return -Inf();
- } else {
- return TInt128(val);
- }
-}
-
-template<typename TValue>
-inline TValue ToYtDecimal(TInt128 val) {
- static_assert(std::is_same<TInt128, TValue>::value || std::is_signed<TValue>::value, "Expected signed value");
- if (IsNormal(val)) {
- return (TValue)val;
- } else if (val == Inf()) {
- return YtDecimalInf<TValue>();
- } else if (val == -Inf()) {
- return -YtDecimalInf<TValue>();
- }
- return YtDecimalNan<TValue>();
-}
-
+template<typename TValue>
+inline constexpr TValue YtDecimalNan() {
+ return std::numeric_limits<TValue>::max();
+}
+
+template<>
+inline constexpr TInt128 YtDecimalNan<TInt128>() {
+ return ~(TInt128(1) << 127);
+}
+
+template<typename TValue>
+inline constexpr TValue YtDecimalInf() {
+ return YtDecimalNan<TValue>() - 1;
+}
+
+template<typename TValue>
+inline TInt128 FromYtDecimal(TValue val) {
+ static_assert(std::is_same<TInt128, TValue>::value || std::is_signed<TValue>::value, "Expected signed value");
+ if (YtDecimalNan<TValue>() == val) {
+ return Nan();
+ } else if (YtDecimalInf<TValue>() == val) {
+ return Inf();
+ } else if (-YtDecimalInf<TValue>() == val) {
+ return -Inf();
+ } else {
+ return TInt128(val);
+ }
+}
+
+template<typename TValue>
+inline TValue ToYtDecimal(TInt128 val) {
+ static_assert(std::is_same<TInt128, TValue>::value || std::is_signed<TValue>::value, "Expected signed value");
+ if (IsNormal(val)) {
+ return (TValue)val;
+ } else if (val == Inf()) {
+ return YtDecimalInf<TValue>();
+ } else if (val == -Inf()) {
+ return -YtDecimalInf<TValue>();
+ }
+ return YtDecimalNan<TValue>();
+}
+
inline TInt128 FromHalfs(ui64 lo, i64 hi) {
ui64 half[2] = {lo, static_cast<ui64>(hi)};
TInt128 val128;
diff --git a/ydb/library/yql/public/decimal/yql_decimal_serialize.cpp b/ydb/library/yql/public/decimal/yql_decimal_serialize.cpp
index 58d1681fe6..7a32197c39 100644
--- a/ydb/library/yql/public/decimal/yql_decimal_serialize.cpp
+++ b/ydb/library/yql/public/decimal/yql_decimal_serialize.cpp
@@ -1,82 +1,82 @@
-#include "yql_decimal_serialize.h"
-
-#include <utility>
-
-namespace NYql {
-namespace NDecimal {
-
-size_t Serialize(TInt128 value, char* buf) {
- if (value == -Nan()) {
- *buf = 0x00;
- return 1U;
- }
-
- if (value == -Inf()) {
- *buf = 0x01;
- return 1U;
- }
-
- if (value == +Inf()) {
- *buf = 0xFE;
- return 1U;
- }
-
- if (value == +Nan()) {
- *buf = 0xFF;
- return 1U;
- }
-
- auto size = sizeof(value);
- auto p = reinterpret_cast<const char*>(&value) + size - 1U;
-
- if (*(p - 1U) & 0x80) {
- while (size > 1U && ~0 == *--p)
- --size;
- *buf = 0x80 - size;
- } else {
- while (size > 1U && 0 == *--p)
- --size;
- *buf = 0x7F + size;
- }
-
- for (auto i = 1U; i < size; ++i) {
- *++buf = *p--;
- }
-
- return size;
-}
-
-std::pair<TInt128, size_t> Deserialize(const char* b) {
- if (!b)
- return std::make_pair(Err(), 0U);
-
- const auto mark = ui8(*b);
- switch (mark) {
- case 0x00u: return std::make_pair(-Nan(), 1U);
- case 0x01u: return std::make_pair(-Inf(), 1U);
- case 0xFEu: return std::make_pair(+Inf(), 1U);
- case 0xFFu: return std::make_pair(+Nan(), 1U);
- }
-
- if (mark < 0x70u || mark > 0x8Fu) {
- return std::make_pair(Err(), 0U);
- }
-
- const bool neg = mark < 0x80u;
- const auto used = neg ? 0x80u - mark : mark - 0x7Fu;
-
- TInt128 v;
- const auto size = sizeof(v);
- auto p = reinterpret_cast<char*>(&v) + size;
-
- for (auto fill = size - used + 2U; --fill;)
- *--p = neg ? ~0 : 0;
-
- for (auto copy = used; --copy;)
- *--p = *++b;
-
- return std::make_pair(v, used);
-}
-
-}
-}
+#include "yql_decimal_serialize.h"
+
+#include <utility>
+
+namespace NYql {
+namespace NDecimal {
+
+size_t Serialize(TInt128 value, char* buf) {
+ if (value == -Nan()) {
+ *buf = 0x00;
+ return 1U;
+ }
+
+ if (value == -Inf()) {
+ *buf = 0x01;
+ return 1U;
+ }
+
+ if (value == +Inf()) {
+ *buf = 0xFE;
+ return 1U;
+ }
+
+ if (value == +Nan()) {
+ *buf = 0xFF;
+ return 1U;
+ }
+
+ auto size = sizeof(value);
+ auto p = reinterpret_cast<const char*>(&value) + size - 1U;
+
+ if (*(p - 1U) & 0x80) {
+ while (size > 1U && ~0 == *--p)
+ --size;
+ *buf = 0x80 - size;
+ } else {
+ while (size > 1U && 0 == *--p)
+ --size;
+ *buf = 0x7F + size;
+ }
+
+ for (auto i = 1U; i < size; ++i) {
+ *++buf = *p--;
+ }
+
+ return size;
+}
+
+std::pair<TInt128, size_t> Deserialize(const char* b) {
+ if (!b)
+ return std::make_pair(Err(), 0U);
+
+ const auto mark = ui8(*b);
+ switch (mark) {
+ case 0x00u: return std::make_pair(-Nan(), 1U);
+ case 0x01u: return std::make_pair(-Inf(), 1U);
+ case 0xFEu: return std::make_pair(+Inf(), 1U);
+ case 0xFFu: return std::make_pair(+Nan(), 1U);
+ }
+
+ if (mark < 0x70u || mark > 0x8Fu) {
+ return std::make_pair(Err(), 0U);
+ }
+
+ const bool neg = mark < 0x80u;
+ const auto used = neg ? 0x80u - mark : mark - 0x7Fu;
+
+ TInt128 v;
+ const auto size = sizeof(v);
+ auto p = reinterpret_cast<char*>(&v) + size;
+
+ for (auto fill = size - used + 2U; --fill;)
+ *--p = neg ? ~0 : 0;
+
+ for (auto copy = used; --copy;)
+ *--p = *++b;
+
+ return std::make_pair(v, used);
+}
+
+}
+}
diff --git a/ydb/library/yql/public/decimal/yql_decimal_serialize.h b/ydb/library/yql/public/decimal/yql_decimal_serialize.h
index 533af2df36..d0ab42a500 100644
--- a/ydb/library/yql/public/decimal/yql_decimal_serialize.h
+++ b/ydb/library/yql/public/decimal/yql_decimal_serialize.h
@@ -1,13 +1,13 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/public/decimal/yql_decimal.h>
-
-namespace NYql {
-namespace NDecimal {
-
-// big-endian 16 bytes buffer.
-size_t Serialize(TInt128 v, char* buff);
-std::pair<TInt128, size_t> Deserialize(const char* buff);
-
-}
-}
+
+namespace NYql {
+namespace NDecimal {
+
+// big-endian 16 bytes buffer.
+size_t Serialize(TInt128 v, char* buff);
+std::pair<TInt128, size_t> Deserialize(const char* buff);
+
+}
+}
diff --git a/ydb/library/yql/public/issue/ut/ya.make b/ydb/library/yql/public/issue/ut/ya.make
index 4fc5841a51..403d17769c 100644
--- a/ydb/library/yql/public/issue/ut/ya.make
+++ b/ydb/library/yql/public/issue/ut/ya.make
@@ -7,7 +7,7 @@ FORK_SUBTESTS()
SRCS(
yql_issue_ut.cpp
yql_issue_manager_ut.cpp
- yql_issue_utils_ut.cpp
+ yql_issue_utils_ut.cpp
yql_warning_ut.cpp
)
diff --git a/ydb/library/yql/public/issue/ya.make b/ydb/library/yql/public/issue/ya.make
index 04a84bee88..4ceb6cddf1 100644
--- a/ydb/library/yql/public/issue/ya.make
+++ b/ydb/library/yql/public/issue/ya.make
@@ -9,7 +9,7 @@ SRCS(
yql_issue.cpp
yql_issue_message.cpp
yql_issue_manager.cpp
- yql_issue_utils.cpp
+ yql_issue_utils.cpp
yql_warning.cpp
)
@@ -19,7 +19,7 @@ PEERDIR(
library/cpp/resource
ydb/public/api/protos
ydb/library/yql/public/issue/protos
- ydb/library/yql/utils
+ ydb/library/yql/utils
)
GENERATE_ENUM_SERIALIZATION(yql_warning.h)
diff --git a/ydb/library/yql/public/issue/yql_issue.cpp b/ydb/library/yql/public/issue/yql_issue.cpp
index 3b89156874..87c31a660a 100644
--- a/ydb/library/yql/public/issue/yql_issue.cpp
+++ b/ydb/library/yql/public/issue/yql_issue.cpp
@@ -1,10 +1,10 @@
-#include "yql_issue.h"
-#include "yql_issue_id.h"
-
-#include <ydb/library/yql/utils/utf8.h>
-
+#include "yql_issue.h"
+#include "yql_issue_id.h"
+
+#include <ydb/library/yql/utils/utf8.h>
+
#include <library/cpp/colorizer/output.h>
-
+
#include <util/string/ascii.h>
#include <util/string/split.h>
#include <util/string/strip.h>
@@ -17,14 +17,14 @@
namespace NYql {
-void SanitizeNonAscii(TString& s) {
- if (!NYql::IsUtf8(s)) {
- for (size_t i = 0; i < s.size(); ++i) {
- if (!IsAscii(s[i])) {
- s[i] = '?';
- }
- }
- }
+void SanitizeNonAscii(TString& s) {
+ if (!NYql::IsUtf8(s)) {
+ for (size_t i = 0; i < s.size(); ++i) {
+ if (!IsAscii(s[i])) {
+ s[i] = '?';
+ }
+ }
+ }
}
TTextWalker& TTextWalker::Advance(char c) {
diff --git a/ydb/library/yql/public/issue/yql_issue.h b/ydb/library/yql/public/issue/yql_issue.h
index 7b97674c9d..e48b1ad638 100644
--- a/ydb/library/yql/public/issue/yql_issue.h
+++ b/ydb/library/yql/public/issue/yql_issue.h
@@ -16,7 +16,7 @@
namespace NYql {
-void SanitizeNonAscii(TString& s);
+void SanitizeNonAscii(TString& s);
///////////////////////////////////////////////////////////////////////////////
// TPosition
@@ -33,7 +33,7 @@ struct TPosition {
, Row(row)
, File(file)
{
- SanitizeNonAscii(File);
+ SanitizeNonAscii(File);
}
explicit operator bool() const {
@@ -124,7 +124,7 @@ public:
, EndPosition(TPosition())
, Message(message)
{
- SanitizeNonAscii(Message);
+ SanitizeNonAscii(Message);
}
template <typename T>
@@ -133,7 +133,7 @@ public:
, EndPosition(position)
, Message(message)
{
- SanitizeNonAscii(Message);
+ SanitizeNonAscii(Message);
}
inline TRange Range() const {
@@ -146,7 +146,7 @@ public:
, EndPosition(endPosition)
, Message(message)
{
- SanitizeNonAscii(Message);
+ SanitizeNonAscii(Message);
}
inline bool operator==(const TIssue& other) const {
diff --git a/ydb/library/yql/public/issue/yql_issue_manager.cpp b/ydb/library/yql/public/issue/yql_issue_manager.cpp
index 7375d2d30f..73495a8b8b 100644
--- a/ydb/library/yql/public/issue/yql_issue_manager.cpp
+++ b/ydb/library/yql/public/issue/yql_issue_manager.cpp
@@ -1,7 +1,7 @@
#include "yql_issue_manager.h"
#include <util/string/cast.h>
-#include <util/string/builder.h>
+#include <util/string/builder.h>
using namespace NYql;
@@ -34,21 +34,21 @@ void TIssueManager::LeaveScope() {
return;
}
- auto subIssue = *RawIssues_.top().first;
- if (subIssue->GetSubIssues().size() == 1) {
- auto nestedIssue = subIssue->GetSubIssues().front();
- if (!nestedIssue->GetSubIssues().empty() && nestedIssue->Position == subIssue->Position && nestedIssue->EndPosition == subIssue->EndPosition) {
- auto msg = subIssue->Message;
- if (nestedIssue->Message) {
- if (msg) {
- msg.append(", ");
- }
- msg.append(nestedIssue->Message);
- }
- subIssue = nestedIssue;
- subIssue->Message = msg;
- }
- }
+ auto subIssue = *RawIssues_.top().first;
+ if (subIssue->GetSubIssues().size() == 1) {
+ auto nestedIssue = subIssue->GetSubIssues().front();
+ if (!nestedIssue->GetSubIssues().empty() && nestedIssue->Position == subIssue->Position && nestedIssue->EndPosition == subIssue->EndPosition) {
+ auto msg = subIssue->Message;
+ if (nestedIssue->Message) {
+ if (msg) {
+ msg.append(", ");
+ }
+ msg.append(nestedIssue->Message);
+ }
+ subIssue = nestedIssue;
+ subIssue->Message = msg;
+ }
+ }
RawIssues_.pop();
if (RawIssues_.top().first.Empty()) {
RawIssues_.top().first = RawIssues_.top().second();
@@ -56,16 +56,16 @@ void TIssueManager::LeaveScope() {
RawIssues_.top().first = new TIssue();
(*RawIssues_.top().first)->SetCode(Max<ui32>(), ESeverity::TSeverityIds_ESeverityId_S_INFO);
} else {
- (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO;
- }
+ (*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO;
+ }
}
- if (subIssue->GetCode() == Max<ui32>()) {
- for (const auto& nestedIssue : subIssue->GetSubIssues()) {
+ if (subIssue->GetCode() == Max<ui32>()) {
+ for (const auto& nestedIssue : subIssue->GetSubIssues()) {
RawIssues_.top().first->Get()->AddSubIssue(nestedIssue);
}
} else {
- RawIssues_.top().first->Get()->AddSubIssue(subIssue);
+ RawIssues_.top().first->Get()->AddSubIssue(subIssue);
}
}
@@ -75,30 +75,30 @@ void TIssueManager::LeaveAllScopes() {
}
}
-TIssuePtr TIssueManager::CheckUniqAndLimit(const TIssue& issue) {
- const auto severity = issue.GetSeverity();
- if (OverflowIssues_[severity]) {
- return {};
- }
- TIssuePtr p = MakeIntrusive<TIssue>(issue);
- if (UniqueIssues_[severity].contains(p)) {
- return {};
- }
- if (IssueLimit_ && UniqueIssues_[severity].size() == IssueLimit_) {
- OverflowIssues_[severity] = MakeIntrusive<TIssue>(TStringBuilder()
- << "Too many " << SeverityToString(issue.GetSeverity()) << " issues");
- OverflowIssues_[severity]->Severity = severity;
- return {};
- }
- UniqueIssues_[severity].insert(p);
- return p;
-}
-
+TIssuePtr TIssueManager::CheckUniqAndLimit(const TIssue& issue) {
+ const auto severity = issue.GetSeverity();
+ if (OverflowIssues_[severity]) {
+ return {};
+ }
+ TIssuePtr p = MakeIntrusive<TIssue>(issue);
+ if (UniqueIssues_[severity].contains(p)) {
+ return {};
+ }
+ if (IssueLimit_ && UniqueIssues_[severity].size() == IssueLimit_) {
+ OverflowIssues_[severity] = MakeIntrusive<TIssue>(TStringBuilder()
+ << "Too many " << SeverityToString(issue.GetSeverity()) << " issues");
+ OverflowIssues_[severity]->Severity = severity;
+ return {};
+ }
+ UniqueIssues_[severity].insert(p);
+ return p;
+}
+
void TIssueManager::RaiseIssue(const TIssue& issue) {
- TIssuePtr p = CheckUniqAndLimit(issue);
- if (!p) {
- return;
- }
+ TIssuePtr p = CheckUniqAndLimit(issue);
+ if (!p) {
+ return;
+ }
if (RawIssues_.empty()) {
CompletedIssues_.AddIssue(issue);
return;
@@ -112,7 +112,7 @@ void TIssueManager::RaiseIssue(const TIssue& issue) {
(*RawIssues_.top().first)->Severity = ESeverity::TSeverityIds_ESeverityId_S_INFO;
}
}
- RawIssues_.top().first->Get()->AddSubIssue(p);
+ RawIssues_.top().first->Get()->AddSubIssue(p);
}
void TIssueManager::RaiseIssues(const TIssues& issues) {
@@ -154,52 +154,52 @@ bool TIssueManager::HasOpenScopes() const {
return !RawIssues_.empty();
}
-TIssues TIssueManager::GetIssues() {
+TIssues TIssueManager::GetIssues() {
LeaveAllScopes();
- return GetCompletedIssues();
+ return GetCompletedIssues();
}
-TIssues TIssueManager::GetCompletedIssues() const {
- TIssues res;
- for (auto& p: OverflowIssues_) {
- if (p) {
- res.AddIssue(*p);
- }
- }
- res.AddIssues(CompletedIssues_);
- return res;
+TIssues TIssueManager::GetCompletedIssues() const {
+ TIssues res;
+ for (auto& p: OverflowIssues_) {
+ if (p) {
+ res.AddIssue(*p);
+ }
+ }
+ res.AddIssues(CompletedIssues_);
+ return res;
}
-void TIssueManager::AddIssues(const TIssues& issues) {
- for (auto& issue: issues) {
- if (auto p = CheckUniqAndLimit(issue)) {
- CompletedIssues_.AddIssue(*p);
- }
- }
+void TIssueManager::AddIssues(const TIssues& issues) {
+ for (auto& issue: issues) {
+ if (auto p = CheckUniqAndLimit(issue)) {
+ CompletedIssues_.AddIssue(*p);
+ }
+ }
}
void TIssueManager::AddIssues(const TPosition& pos, const TIssues& issues) {
- for (auto& issue: issues) {
- if (auto p = CheckUniqAndLimit(TIssue(pos, issue.Message))) {
- CompletedIssues_.AddIssue(*p);
- }
- }
-}
-
-void TIssueManager::Reset(const TIssues& issues) {
- for (auto& p: OverflowIssues_) {
- p.Drop();
- }
-
- for (auto& s: UniqueIssues_) {
- s.clear();
- }
- CompletedIssues_.Clear();
-
+ for (auto& issue: issues) {
+ if (auto p = CheckUniqAndLimit(TIssue(pos, issue.Message))) {
+ CompletedIssues_.AddIssue(*p);
+ }
+ }
+}
+
+void TIssueManager::Reset(const TIssues& issues) {
+ for (auto& p: OverflowIssues_) {
+ p.Drop();
+ }
+
+ for (auto& s: UniqueIssues_) {
+ s.clear();
+ }
+ CompletedIssues_.Clear();
+
while (!RawIssues_.empty()) {
RawIssues_.pop();
}
- AddIssues(issues);
+ AddIssues(issues);
}
void TIssueManager::Reset() {
diff --git a/ydb/library/yql/public/issue/yql_issue_manager.h b/ydb/library/yql/public/issue/yql_issue_manager.h
index 9ad5ac7bb4..6bb70e99fc 100644
--- a/ydb/library/yql/public/issue/yql_issue_manager.h
+++ b/ydb/library/yql/public/issue/yql_issue_manager.h
@@ -1,13 +1,13 @@
#pragma once
-#include "yql_issue.h"
-#include "yql_warning.h"
-
+#include "yql_issue.h"
+#include "yql_warning.h"
+
#include <util/generic/maybe.h>
#include <util/generic/stack.h>
-#include <util/generic/hash_set.h>
+#include <util/generic/hash_set.h>
-#include <array>
+#include <array>
namespace NYql {
@@ -23,39 +23,39 @@ public:
void AddIssues(const TPosition& pos, const TIssues& issues);
bool HasOpenScopes() const;
- TIssues GetIssues();
- TIssues GetCompletedIssues() const;
+ TIssues GetIssues();
+ TIssues GetCompletedIssues() const;
- void Reset(const TIssues& issues);
+ void Reset(const TIssues& issues);
void Reset();
void AddWarningRule(const TWarningRule &rule);
void SetWarningToErrorTreatMessage(const TString& msg);
-
- void SetIssueCountLimit(size_t limit) {
- IssueLimit_ = limit;
- }
-
+
+ void SetIssueCountLimit(size_t limit) {
+ IssueLimit_ = limit;
+ }
+
private:
- TIssuePtr CheckUniqAndLimit(const TIssue& issue);
-
- struct TIssueHash {
- ui64 operator()(const TIssuePtr& p) {
- return p->Hash();
- }
- };
- struct TIssueEqual {
- bool operator()(const TIssuePtr& l, const TIssuePtr& r) {
- return *l == *r;
- }
- };
+ TIssuePtr CheckUniqAndLimit(const TIssue& issue);
+
+ struct TIssueHash {
+ ui64 operator()(const TIssuePtr& p) {
+ return p->Hash();
+ }
+ };
+ struct TIssueEqual {
+ bool operator()(const TIssuePtr& l, const TIssuePtr& r) {
+ return *l == *r;
+ }
+ };
TStack<std::pair<TMaybe<TIssuePtr>, std::function<TIssuePtr()>>> RawIssues_;
TIssues CompletedIssues_;
TMaybe<TString> WarningToErrorTreatMessage_;
TWarningPolicy WarningPolicy_;
- std::array<TIssuePtr, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> OverflowIssues_;
- std::array<THashSet<TIssuePtr, TIssueHash, TIssueEqual>, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> UniqueIssues_;
- size_t IssueLimit_ = 0;
+ std::array<TIssuePtr, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> OverflowIssues_;
+ std::array<THashSet<TIssuePtr, TIssueHash, TIssueEqual>, NYql::TSeverityIds::ESeverityId_ARRAYSIZE> UniqueIssues_;
+ size_t IssueLimit_ = 0;
};
class TIssueScopeGuard: private TNonCopyable {
diff --git a/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp b/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp
index 287f63fedd..92abc9cef0 100644
--- a/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp
+++ b/ydb/library/yql/public/issue/yql_issue_manager_ut.cpp
@@ -4,9 +4,9 @@
using namespace NYql;
-static std::function<TIssuePtr()> CreateScopeIssueFunction(TString name, ui32 column, ui32 row) {
- return [name, column, row]() {
- return new TIssue(TPosition(column, row), name);
+static std::function<TIssuePtr()> CreateScopeIssueFunction(TString name, ui32 column, ui32 row) {
+ return [name, column, row]() {
+ return new TIssue(TPosition(column, row), name);
};
}
@@ -21,7 +21,7 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) {
Y_UNIT_TEST(NoErrorOneLevelTest) {
TIssueManager issueManager;
- issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
auto completedIssues = issueManager.GetCompletedIssues();
UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0);
issueManager.LeaveScope();
@@ -31,8 +31,8 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) {
Y_UNIT_TEST(NoErrorTwoLevelsTest) {
TIssueManager issueManager;
- issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
- issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
issueManager.LeaveScope();
auto completedIssues = issueManager.GetCompletedIssues();
UNIT_ASSERT_VALUES_EQUAL(completedIssues.Size(), 0);
@@ -43,7 +43,7 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) {
Y_UNIT_TEST(OneErrorOneLevelTest) {
TIssueManager issueManager;
- issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
auto completedIssues1 = issueManager.GetCompletedIssues();
UNIT_ASSERT_VALUES_EQUAL(completedIssues1.Size(), 0);
issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
@@ -64,8 +64,8 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) {
Y_UNIT_TEST(OneErrorTwoLevelsTest) {
TIssueManager issueManager;
- issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
- issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
issueManager.LeaveScope();
issueManager.LeaveScope();
@@ -83,10 +83,10 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) {
Y_UNIT_TEST(MultiErrorsMultiLevelsTest) {
TIssueManager issueManager;
- issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
issueManager.RaiseIssue(TIssue(TPosition(), "WarningScope1"));
- issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
- issueManager.AddScope(CreateScopeIssueFunction("C", 2, 2));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
+ issueManager.AddScope(CreateScopeIssueFunction("C", 2, 2));
issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope3"));
issueManager.LeaveScope();
issueManager.RaiseIssue(TIssue(TPosition(), "ErrorScope2"));
@@ -105,8 +105,8 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) {
UNIT_ASSERT_VALUES_EQUAL(subIssues[1]->GetSubIssues()[1]->Message, "ErrorScope2");
auto ref = R"___(<main>: Error: A
<main>: Error: WarningScope1
- <main>:1:1: Error: B
- <main>:2:2: Error: C
+ <main>:1:1: Error: B
+ <main>:2:2: Error: C
<main>: Error: ErrorScope3
<main>: Error: ErrorScope2
)___";
@@ -116,7 +116,7 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) {
Y_UNIT_TEST(TIssueScopeGuardSimpleTest) {
TIssueManager issueManager;
{
- TIssueScopeGuard guard(issueManager, CreateScopeIssueFunction("A", 0, 0));
+ TIssueScopeGuard guard(issueManager, CreateScopeIssueFunction("A", 0, 0));
issueManager.RaiseIssue(TIssue(TPosition(1,2), "ErrorScope1"));
}
auto issues = issueManager.GetIssues();
@@ -127,80 +127,80 @@ Y_UNIT_TEST_SUITE(TIssueManagerTest) {
UNIT_ASSERT_VALUES_EQUAL(subIssues.size(), 1);
UNIT_ASSERT_VALUES_EQUAL(subIssues[0]->Message, "ErrorScope1");
}
-
- Y_UNIT_TEST(FuseScopesTest) {
- TIssueManager issueManager;
- issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
- issueManager.AddScope(CreateScopeIssueFunction("B", 0, 0));
- issueManager.AddScope(CreateScopeIssueFunction("C", 0, 0));
- issueManager.AddScope(CreateScopeIssueFunction("D", 0, 0));
- issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
- issueManager.LeaveScope();
- issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo"));
- issueManager.LeaveScope();
- issueManager.LeaveScope();
- issueManager.LeaveScope();
- auto issues = issueManager.GetIssues();
- auto ref = R"___(<main>: Error: A
- <main>: Error: B, C
- <main>: Error: D
- <main>:2:1: Error: IssueOne
- <main>:2:1: Error: IssueTwo
-)___";
- UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref);
- }
-
- Y_UNIT_TEST(UniqIssues) {
- TIssueManager issueManager;
- issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
- issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
- issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
- issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo"));
- issueManager.RaiseIssue(TIssue(TPosition(2,3), "IssueOne"));
- issueManager.RaiseWarning(TIssue(TPosition(2,3), "IssueOne").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
- issueManager.LeaveScope();
- issueManager.LeaveScope();
- issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
- auto issues = issueManager.GetIssues();
- UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1);
- auto ref = R"___(<main>: Error: A
- <main>:1:1: Error: B
- <main>:2:1: Error: IssueOne
- <main>:2:1: Error: IssueTwo
- <main>:3:2: Error: IssueOne
- <main>:3:2: Warning: IssueOne, code: 1
-)___";
- UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref);
- }
-
- Y_UNIT_TEST(Limits) {
- TIssueManager issueManager;
- issueManager.SetIssueCountLimit(2);
- issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
- issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
- issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue1"));
- issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue2"));
- issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue3"));
- issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue4").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
- issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue5").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
- issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue6").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
- issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue7").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO));
- issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue8").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO));
- issueManager.LeaveScope();
- issueManager.LeaveScope();
- auto issues = issueManager.GetIssues();
- UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 3);
- auto ref = R"___(<main>: Error: Too many Error issues
-<main>: Warning: Too many Warning issues
-<main>: Error: A
- <main>:1:1: Error: B
- <main>:1:1: Error: Issue1
- <main>:1:1: Error: Issue2
- <main>:1:1: Warning: Issue4, code: 1
- <main>:1:1: Warning: Issue5, code: 1
- <main>:1:1: Info: Issue7, code: 2
- <main>:1:1: Info: Issue8, code: 2
-)___";
- UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref);
- }
+
+ Y_UNIT_TEST(FuseScopesTest) {
+ TIssueManager issueManager;
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("C", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("D", 0, 0));
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
+ issueManager.LeaveScope();
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo"));
+ issueManager.LeaveScope();
+ issueManager.LeaveScope();
+ issueManager.LeaveScope();
+ auto issues = issueManager.GetIssues();
+ auto ref = R"___(<main>: Error: A
+ <main>: Error: B, C
+ <main>: Error: D
+ <main>:2:1: Error: IssueOne
+ <main>:2:1: Error: IssueTwo
+)___";
+ UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref);
+ }
+
+ Y_UNIT_TEST(UniqIssues) {
+ TIssueManager issueManager;
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueTwo"));
+ issueManager.RaiseIssue(TIssue(TPosition(2,3), "IssueOne"));
+ issueManager.RaiseWarning(TIssue(TPosition(2,3), "IssueOne").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
+ issueManager.LeaveScope();
+ issueManager.LeaveScope();
+ issueManager.RaiseIssue(TIssue(TPosition(1,2), "IssueOne"));
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 1);
+ auto ref = R"___(<main>: Error: A
+ <main>:1:1: Error: B
+ <main>:2:1: Error: IssueOne
+ <main>:2:1: Error: IssueTwo
+ <main>:3:2: Error: IssueOne
+ <main>:3:2: Warning: IssueOne, code: 1
+)___";
+ UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref);
+ }
+
+ Y_UNIT_TEST(Limits) {
+ TIssueManager issueManager;
+ issueManager.SetIssueCountLimit(2);
+ issueManager.AddScope(CreateScopeIssueFunction("A", 0, 0));
+ issueManager.AddScope(CreateScopeIssueFunction("B", 1, 1));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue1"));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue2"));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue3"));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue4").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue5").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue6").SetCode(1, ESeverity::TSeverityIds_ESeverityId_S_WARNING));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue7").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO));
+ issueManager.RaiseIssue(TIssue(TPosition(1,1), "Issue8").SetCode(2, ESeverity::TSeverityIds_ESeverityId_S_INFO));
+ issueManager.LeaveScope();
+ issueManager.LeaveScope();
+ auto issues = issueManager.GetIssues();
+ UNIT_ASSERT_VALUES_EQUAL(issues.Size(), 3);
+ auto ref = R"___(<main>: Error: Too many Error issues
+<main>: Warning: Too many Warning issues
+<main>: Error: A
+ <main>:1:1: Error: B
+ <main>:1:1: Error: Issue1
+ <main>:1:1: Error: Issue2
+ <main>:1:1: Warning: Issue4, code: 1
+ <main>:1:1: Warning: Issue5, code: 1
+ <main>:1:1: Info: Issue7, code: 2
+ <main>:1:1: Info: Issue8, code: 2
+)___";
+ UNIT_ASSERT_VALUES_EQUAL(issues.ToString(), ref);
+ }
}
diff --git a/ydb/library/yql/public/issue/yql_issue_message.cpp b/ydb/library/yql/public/issue/yql_issue_message.cpp
index f9d512ca5a..50127ad986 100644
--- a/ydb/library/yql/public/issue/yql_issue_message.cpp
+++ b/ydb/library/yql/public/issue/yql_issue_message.cpp
@@ -1,12 +1,12 @@
-#include "yql_issue_message.h"
-
+#include "yql_issue_message.h"
+
#include <ydb/library/yql/public/issue/protos/issue_message.pb.h>
#include <ydb/public/api/protos/ydb_issue_message.pb.h>
+
+#include <util/generic/deque.h>
+#include <util/generic/yexception.h>
-#include <util/generic/deque.h>
-#include <util/generic/yexception.h>
-
-#include <tuple>
+#include <tuple>
namespace NYql {
@@ -15,7 +15,7 @@ using namespace NIssue::NProto;
template<typename TIssueMessage>
TIssue IssueFromMessage(const TIssueMessage& issueMessage) {
TIssue topIssue;
- TDeque<std::pair<TIssue*, const TIssueMessage*>> queue;
+ TDeque<std::pair<TIssue*, const TIssueMessage*>> queue;
queue.push_front(std::make_pair(&topIssue, &issueMessage));
while (!queue.empty()) {
TIssue& issue = *queue.back().first;
@@ -55,43 +55,43 @@ void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage>
}
template<typename TIssueMessage>
-void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) {
- TDeque<std::pair<const TIssue*, TIssueMessage*>> queue;
- queue.push_front(std::make_pair(&topIssue, issueMessage));
- while (!queue.empty()) {
- const TIssue& issue = *queue.back().first;
- auto& message = *queue.back().second;
- queue.pop_back();
- if (issue.Position) {
- auto& position = *message.mutable_position();
- position.set_row(issue.Position.Row);
- position.set_column(issue.Position.Column);
- position.set_file(issue.Position.File);
- }
- if (issue.EndPosition) {
- auto& endPosition = *message.mutable_end_position();
- endPosition.set_row(issue.EndPosition.Row);
- endPosition.set_column(issue.EndPosition.Column);
- }
- message.set_message(issue.Message);
- message.set_issue_code(issue.GetCode());
- message.set_severity(issue.GetSeverity());
-
- for (auto subIssue : issue.GetSubIssues()) {
- TIssueMessage* subMessage = message.add_issues();
- queue.push_front(std::make_pair(subIssue.Get(), subMessage));
- }
+void IssueToMessage(const TIssue& topIssue, TIssueMessage* issueMessage) {
+ TDeque<std::pair<const TIssue*, TIssueMessage*>> queue;
+ queue.push_front(std::make_pair(&topIssue, issueMessage));
+ while (!queue.empty()) {
+ const TIssue& issue = *queue.back().first;
+ auto& message = *queue.back().second;
+ queue.pop_back();
+ if (issue.Position) {
+ auto& position = *message.mutable_position();
+ position.set_row(issue.Position.Row);
+ position.set_column(issue.Position.Column);
+ position.set_file(issue.Position.File);
+ }
+ if (issue.EndPosition) {
+ auto& endPosition = *message.mutable_end_position();
+ endPosition.set_row(issue.EndPosition.Row);
+ endPosition.set_column(issue.EndPosition.Column);
+ }
+ message.set_message(issue.Message);
+ message.set_issue_code(issue.GetCode());
+ message.set_severity(issue.GetSeverity());
+
+ for (auto subIssue : issue.GetSubIssues()) {
+ TIssueMessage* subMessage = message.add_issues();
+ queue.push_front(std::make_pair(subIssue.Get(), subMessage));
+ }
}
}
template<typename TIssueMessage>
-void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage> *message) {
+void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage> *message) {
message->Clear();
if (!issues)
return;
message->Reserve(issues.Size());
for (const auto &issue : issues) {
- IssueToMessage(issue, message->Add());
+ IssueToMessage(issue, message->Add());
}
}
@@ -101,30 +101,30 @@ template
TIssue IssueFromMessage<NYql::NIssue::NProto::IssueMessage>(const NYql::NIssue::NProto::IssueMessage& issueMessage);
template
-void IssuesFromMessage<Ydb::Issue::IssueMessage>(const ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& message, TIssues& issues);
+void IssuesFromMessage<Ydb::Issue::IssueMessage>(const ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>& message, TIssues& issues);
template
-void IssuesFromMessage<NYql::NIssue::NProto::IssueMessage>(const ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>& message, TIssues& issues);
+void IssuesFromMessage<NYql::NIssue::NProto::IssueMessage>(const ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>& message, TIssues& issues);
template
-void IssueToMessage<Ydb::Issue::IssueMessage>(const TIssue& topIssue, Ydb::Issue::IssueMessage* issueMessage);
+void IssueToMessage<Ydb::Issue::IssueMessage>(const TIssue& topIssue, Ydb::Issue::IssueMessage* issueMessage);
template
-void IssueToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssue& topIssue, NYql::NIssue::NProto::IssueMessage* issueMessage);
+void IssueToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssue& topIssue, NYql::NIssue::NProto::IssueMessage* issueMessage);
template
-void IssuesToMessage<Ydb::Issue::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* message);
+void IssuesToMessage<Ydb::Issue::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<Ydb::Issue::IssueMessage>* message);
template
-void IssuesToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>* message);
+void IssuesToMessage<NYql::NIssue::NProto::IssueMessage>(const TIssues& issues, ::google::protobuf::RepeatedPtrField<NYql::NIssue::NProto::IssueMessage>* message);
-NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue) {
+NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue) {
NIssue::NProto::IssueMessage issueMessage;
- IssueToMessage(topIssue, &issueMessage);
+ IssueToMessage(topIssue, &issueMessage);
return issueMessage;
}
-TString IssueToBinaryMessage(const TIssue& issue) {
+TString IssueToBinaryMessage(const TIssue& issue) {
TString result;
Ydb::Issue::IssueMessage protobuf;
- IssueToMessage(issue, &protobuf);
+ IssueToMessage(issue, &protobuf);
Y_PROTOBUF_SUPPRESS_NODISCARD protobuf.SerializeToString(&result);
return result;
}
diff --git a/ydb/library/yql/public/issue/yql_issue_message.h b/ydb/library/yql/public/issue/yql_issue_message.h
index b9d1ba529c..c3fde67413 100644
--- a/ydb/library/yql/public/issue/yql_issue_message.h
+++ b/ydb/library/yql/public/issue/yql_issue_message.h
@@ -2,8 +2,8 @@
#include "yql_issue.h"
-#include <util/generic/ylimits.h>
-
+#include <util/generic/ylimits.h>
+
namespace NYql {
namespace NIssue {
@@ -17,14 +17,14 @@ TIssue IssueFromMessage(const TIssueMessage& issueMessage);
template<typename TIssueMessage>
void IssuesFromMessage(const ::google::protobuf::RepeatedPtrField<TIssueMessage>& message, TIssues& issues);
-NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue);
+NIssue::NProto::IssueMessage IssueToMessage(const TIssue& topIssue);
template<typename TIssueMessage>
-void IssueToMessage(const TIssue& topIssue, TIssueMessage* message);
+void IssueToMessage(const TIssue& topIssue, TIssueMessage* message);
template<typename TIssueMessage>
-void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage>* message);
+void IssuesToMessage(const TIssues& issues, ::google::protobuf::RepeatedPtrField<TIssueMessage>* message);
-TString IssueToBinaryMessage(const TIssue& issue);
+TString IssueToBinaryMessage(const TIssue& issue);
TIssue IssueFromBinaryMessage(const TString& binaryMessage);
}
diff --git a/ydb/library/yql/public/issue/yql_issue_ut.cpp b/ydb/library/yql/public/issue/yql_issue_ut.cpp
index 87b417da39..5852c174ca 100644
--- a/ydb/library/yql/public/issue/yql_issue_ut.cpp
+++ b/ydb/library/yql/public/issue/yql_issue_ut.cpp
@@ -46,13 +46,13 @@ void ensureMessageTypesSame(const Descriptor* a, const Descriptor* b, THashSet<T
Y_UNIT_TEST_SUITE(IssueTest) {
Y_UNIT_TEST(Ascii) {
TIssue issue1("тест abc");
- UNIT_ASSERT_VALUES_EQUAL(issue1.Message, "тест abc");
- TIssue issue2("\xFF abc");
- UNIT_ASSERT_VALUES_EQUAL(issue2.Message, "? abc");
- TIssue issue3("");
- UNIT_ASSERT_VALUES_EQUAL(issue3.Message, "");
- TIssue issue4("abc");
- UNIT_ASSERT_VALUES_EQUAL(issue4.Message, "abc");
+ UNIT_ASSERT_VALUES_EQUAL(issue1.Message, "тест abc");
+ TIssue issue2("\xFF abc");
+ UNIT_ASSERT_VALUES_EQUAL(issue2.Message, "? abc");
+ TIssue issue3("");
+ UNIT_ASSERT_VALUES_EQUAL(issue3.Message, "");
+ TIssue issue4("abc");
+ UNIT_ASSERT_VALUES_EQUAL(issue4.Message, "abc");
}
}
diff --git a/ydb/library/yql/public/issue/yql_issue_utils.cpp b/ydb/library/yql/public/issue/yql_issue_utils.cpp
index 364560ea4d..b8332fd201 100644
--- a/ydb/library/yql/public/issue/yql_issue_utils.cpp
+++ b/ydb/library/yql/public/issue/yql_issue_utils.cpp
@@ -1,114 +1,114 @@
-#include "yql_issue_utils.h"
-
-#include <util/system/yassert.h>
-
-#include <tuple>
-#include <list>
-#include <algorithm>
-#include <deque>
-
-namespace NYql {
-
-TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts) {
- // [issue, level, parent, visibleParent]
- std::list<std::tuple<const TIssue*, ui32, size_t, size_t>> issueQueue;
- // [issue, targetIssue, level, parent, visible, visibleParent, targetSkipIssue]
- std::deque<std::tuple<const TIssue*, TIssue*, ui32, size_t, bool, size_t, TIssue*>> issues;
- // [depth from bottom, position]
- std::list<size_t> leafs;
-
- const auto depthBeforeLeaf = std::max(opts.KeepTailLevels, ui32(1)) - 1;
- const auto maxLevels = std::max(opts.MaxLevels - std::min(opts.MaxLevels, depthBeforeLeaf + 1), ui32(1));
-
- issueQueue.emplace_front(&topIssue, 0, 0, 0);
- while (!issueQueue.empty()) {
- const auto issue = std::get<0>(issueQueue.back());
- const auto level = std::get<1>(issueQueue.back());
- const auto parent = std::get<2>(issueQueue.back());
- const auto visibleParent = std::get<3>(issueQueue.back());
- issueQueue.pop_back();
-
- const bool visible = issue->GetSubIssues().empty() || level < maxLevels;
- const auto pos = issues.size();
- issues.emplace_back(issue, nullptr, level, parent, visible, visibleParent, nullptr);
- if (issue->GetSubIssues().empty()) {
- if (level != 0) {
- leafs.push_back(pos);
- }
- } else {
- for (auto subIssue : issue->GetSubIssues()) {
- issueQueue.emplace_front(subIssue.Get(), level + 1, pos, visible ? pos : visibleParent);
- }
- }
- }
-
- if (depthBeforeLeaf && !leafs.empty()) {
- for (size_t pos: leafs) {
- ui32 depth = depthBeforeLeaf;
- auto parent = std::get<3>(issues.at(pos));
- while (depth && parent) {
- auto& visible = std::get<4>(issues.at(parent));
- auto& visibleParent = std::get<5>(issues.at(pos));
- if (!visible || visibleParent != parent) {
- visible = true;
- visibleParent = parent; // Update visible parent
- --depth;
- pos = parent;
- parent = std::get<3>(issues.at(parent));
- } else {
- break;
- }
- }
- }
- }
- leafs.clear();
-
- auto copyIssue = [](TIssue& target, const TIssue& src) {
- target.Message = src.Message;
- target.IssueCode = src.IssueCode;
- target.Severity = src.Severity;
- target.Position = src.Position;
- target.EndPosition = src.EndPosition;
- };
-
- TIssue result;
- for (auto& i: issues) {
- const auto srcIssue = std::get<0>(i);
- auto& targetIssue = std::get<1>(i);
- const auto level = std::get<2>(i);
- const auto parent = std::get<3>(i);
- const auto visible = std::get<4>(i);
- const auto visibleParent = std::get<5>(i);
-
- if (0 == level) {
- targetIssue = &result;
- copyIssue(*targetIssue, *srcIssue);
- } else if (visible) {
- auto& parentRec = issues.at(visibleParent);
- auto& parentTargetIssue = std::get<1>(parentRec);
- if (parent != visibleParent) {
- auto& parentSkipIssue = std::get<6>(parentRec);
- if (!parentSkipIssue) {
- const auto parentIssue = std::get<0>(parentRec);
- auto newIssue = MakeIntrusive<TIssue>("(skipped levels)");
- newIssue->SetCode(parentIssue->GetCode(), parentIssue->GetSeverity());
- parentTargetIssue->AddSubIssue(newIssue);
- parentSkipIssue = newIssue.Get();
- }
- auto newIssue = MakeIntrusive<TIssue>(TString{});
- copyIssue(*newIssue, *srcIssue);
- parentSkipIssue->AddSubIssue(newIssue);
- targetIssue = newIssue.Get();
- } else {
- auto newIssue = MakeIntrusive<TIssue>(TString{});
- copyIssue(*newIssue, *srcIssue);
- parentTargetIssue->AddSubIssue(newIssue);
- targetIssue = newIssue.Get();
- }
- }
- }
- return result;
-}
-
-} // namspace NYql
-
+#include "yql_issue_utils.h"
+
+#include <util/system/yassert.h>
+
+#include <tuple>
+#include <list>
+#include <algorithm>
+#include <deque>
+
+namespace NYql {
+
+TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts) {
+ // [issue, level, parent, visibleParent]
+ std::list<std::tuple<const TIssue*, ui32, size_t, size_t>> issueQueue;
+ // [issue, targetIssue, level, parent, visible, visibleParent, targetSkipIssue]
+ std::deque<std::tuple<const TIssue*, TIssue*, ui32, size_t, bool, size_t, TIssue*>> issues;
+ // [depth from bottom, position]
+ std::list<size_t> leafs;
+
+ const auto depthBeforeLeaf = std::max(opts.KeepTailLevels, ui32(1)) - 1;
+ const auto maxLevels = std::max(opts.MaxLevels - std::min(opts.MaxLevels, depthBeforeLeaf + 1), ui32(1));
+
+ issueQueue.emplace_front(&topIssue, 0, 0, 0);
+ while (!issueQueue.empty()) {
+ const auto issue = std::get<0>(issueQueue.back());
+ const auto level = std::get<1>(issueQueue.back());
+ const auto parent = std::get<2>(issueQueue.back());
+ const auto visibleParent = std::get<3>(issueQueue.back());
+ issueQueue.pop_back();
+
+ const bool visible = issue->GetSubIssues().empty() || level < maxLevels;
+ const auto pos = issues.size();
+ issues.emplace_back(issue, nullptr, level, parent, visible, visibleParent, nullptr);
+ if (issue->GetSubIssues().empty()) {
+ if (level != 0) {
+ leafs.push_back(pos);
+ }
+ } else {
+ for (auto subIssue : issue->GetSubIssues()) {
+ issueQueue.emplace_front(subIssue.Get(), level + 1, pos, visible ? pos : visibleParent);
+ }
+ }
+ }
+
+ if (depthBeforeLeaf && !leafs.empty()) {
+ for (size_t pos: leafs) {
+ ui32 depth = depthBeforeLeaf;
+ auto parent = std::get<3>(issues.at(pos));
+ while (depth && parent) {
+ auto& visible = std::get<4>(issues.at(parent));
+ auto& visibleParent = std::get<5>(issues.at(pos));
+ if (!visible || visibleParent != parent) {
+ visible = true;
+ visibleParent = parent; // Update visible parent
+ --depth;
+ pos = parent;
+ parent = std::get<3>(issues.at(parent));
+ } else {
+ break;
+ }
+ }
+ }
+ }
+ leafs.clear();
+
+ auto copyIssue = [](TIssue& target, const TIssue& src) {
+ target.Message = src.Message;
+ target.IssueCode = src.IssueCode;
+ target.Severity = src.Severity;
+ target.Position = src.Position;
+ target.EndPosition = src.EndPosition;
+ };
+
+ TIssue result;
+ for (auto& i: issues) {
+ const auto srcIssue = std::get<0>(i);
+ auto& targetIssue = std::get<1>(i);
+ const auto level = std::get<2>(i);
+ const auto parent = std::get<3>(i);
+ const auto visible = std::get<4>(i);
+ const auto visibleParent = std::get<5>(i);
+
+ if (0 == level) {
+ targetIssue = &result;
+ copyIssue(*targetIssue, *srcIssue);
+ } else if (visible) {
+ auto& parentRec = issues.at(visibleParent);
+ auto& parentTargetIssue = std::get<1>(parentRec);
+ if (parent != visibleParent) {
+ auto& parentSkipIssue = std::get<6>(parentRec);
+ if (!parentSkipIssue) {
+ const auto parentIssue = std::get<0>(parentRec);
+ auto newIssue = MakeIntrusive<TIssue>("(skipped levels)");
+ newIssue->SetCode(parentIssue->GetCode(), parentIssue->GetSeverity());
+ parentTargetIssue->AddSubIssue(newIssue);
+ parentSkipIssue = newIssue.Get();
+ }
+ auto newIssue = MakeIntrusive<TIssue>(TString{});
+ copyIssue(*newIssue, *srcIssue);
+ parentSkipIssue->AddSubIssue(newIssue);
+ targetIssue = newIssue.Get();
+ } else {
+ auto newIssue = MakeIntrusive<TIssue>(TString{});
+ copyIssue(*newIssue, *srcIssue);
+ parentTargetIssue->AddSubIssue(newIssue);
+ targetIssue = newIssue.Get();
+ }
+ }
+ }
+ return result;
+}
+
+} // namspace NYql
+
diff --git a/ydb/library/yql/public/issue/yql_issue_utils.h b/ydb/library/yql/public/issue/yql_issue_utils.h
index 023ea0edbf..786b7c91b7 100644
--- a/ydb/library/yql/public/issue/yql_issue_utils.h
+++ b/ydb/library/yql/public/issue/yql_issue_utils.h
@@ -1,31 +1,31 @@
-#pragma once
-
-#include "yql_issue.h"
-
-#include <util/generic/ylimits.h>
-#include <util/system/types.h>
-
-namespace NYql {
-
-struct TTruncateIssueOpts {
-
-#define YQL_TRUNC_DECL_FIELD(type, name, def) \
- TTruncateIssueOpts& Set##name(type arg##name)& { \
- name = arg##name; \
- return *this; \
- } \
- TTruncateIssueOpts&& Set##name(type arg##name)&& { \
- name = arg##name; \
- return std::move(*this); \
- } \
- type name = def;
-
- YQL_TRUNC_DECL_FIELD(ui32, MaxLevels, Max<ui32>())
- YQL_TRUNC_DECL_FIELD(ui32, KeepTailLevels, 1)
-
-#undef YQL_TRUNC_DECL_FIELD
-};
-
-TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts = {});
-
-} // namespace NYql
+#pragma once
+
+#include "yql_issue.h"
+
+#include <util/generic/ylimits.h>
+#include <util/system/types.h>
+
+namespace NYql {
+
+struct TTruncateIssueOpts {
+
+#define YQL_TRUNC_DECL_FIELD(type, name, def) \
+ TTruncateIssueOpts& Set##name(type arg##name)& { \
+ name = arg##name; \
+ return *this; \
+ } \
+ TTruncateIssueOpts&& Set##name(type arg##name)&& { \
+ name = arg##name; \
+ return std::move(*this); \
+ } \
+ type name = def;
+
+ YQL_TRUNC_DECL_FIELD(ui32, MaxLevels, Max<ui32>())
+ YQL_TRUNC_DECL_FIELD(ui32, KeepTailLevels, 1)
+
+#undef YQL_TRUNC_DECL_FIELD
+};
+
+TIssue TruncateIssueLevels(const TIssue& topIssue, TTruncateIssueOpts opts = {});
+
+} // namespace NYql
diff --git a/ydb/library/yql/public/issue/yql_issue_utils_ut.cpp b/ydb/library/yql/public/issue/yql_issue_utils_ut.cpp
index 657643a613..add4599682 100644
--- a/ydb/library/yql/public/issue/yql_issue_utils_ut.cpp
+++ b/ydb/library/yql/public/issue/yql_issue_utils_ut.cpp
@@ -1,200 +1,200 @@
-#include "yql_issue_utils.h"
-#include "yql_issue.h"
-#include "yql_issue_id.h"
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/string/subst.h>
-
-using namespace NYql;
-
-Y_UNIT_TEST_SUITE(TIssueUtilsTest) {
- Y_UNIT_TEST(TruncLevels1) {
- auto level0 = MakeIntrusive<TIssue>("level0");
- auto level1 = MakeIntrusive<TIssue>("level1");
- auto level2 = MakeIntrusive<TIssue>("level2");
- auto level30 = MakeIntrusive<TIssue>("level30");
- auto level31 = MakeIntrusive<TIssue>("level31");
- auto level40 = MakeIntrusive<TIssue>("level40");
- auto level41 = MakeIntrusive<TIssue>("level41");
- auto level51 = MakeIntrusive<TIssue>("level51");
-
- /*
- * * 0
- * |
- * * 1
- * |
- * * 2 --
- * | |
- * * 30 * 31
- * | |
- * * 40 * 41
- * |
- * * 51
- */
-
- level0->AddSubIssue(level1);
- level1->AddSubIssue(level2);
- level2->AddSubIssue(level30);
- level2->AddSubIssue(level31);
- level30->AddSubIssue(level40);
- level31->AddSubIssue(level41);
- level41->AddSubIssue(level51);
-
- {
- const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString();
- const auto expected =
-R"___(<main>: Error: level0
- <main>: Error: level1
- <main>: Error: (skipped levels)
- <main>: Error: level30
- <main>: Error: level40
- <main>: Error: level41
- <main>: Error: level51
-)___";
- UNIT_ASSERT_STRINGS_EQUAL(res, expected);
- }
-
- {
- const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(3).SetKeepTailLevels(1))}).ToString();
- const auto expected =
-R"___(<main>: Error: level0
- <main>: Error: level1
- <main>: Error: (skipped levels)
- <main>: Error: level40
- <main>: Error: level51
-)___";
- UNIT_ASSERT_STRINGS_EQUAL(res, expected);
- }
- }
-
- Y_UNIT_TEST(TruncLevels2) {
- auto level0 = MakeIntrusive<TIssue>("level0");
- auto level1 = MakeIntrusive<TIssue>("level1");
- auto level2 = MakeIntrusive<TIssue>("level2");
- auto level3 = MakeIntrusive<TIssue>("level3");
- auto level40 = MakeIntrusive<TIssue>("level40");
- auto level41 = MakeIntrusive<TIssue>("level41");
-
- /*
- * * 0
- * |
- * * 1
- * |
- * * 2
- * |
- * * 3 --
- * | |
- * * 40 * 41
- */
-
- level0->AddSubIssue(level1);
- level1->AddSubIssue(level2);
- level2->AddSubIssue(level3);
- level3->AddSubIssue(level40);
- level3->AddSubIssue(level41);
-
- {
- const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString();
- const auto expected =
-R"___(<main>: Error: level0
- <main>: Error: level1
- <main>: Error: (skipped levels)
- <main>: Error: level3
- <main>: Error: level40
- <main>: Error: level41
-)___";
- UNIT_ASSERT_STRINGS_EQUAL(res, expected);
- }
- }
-
- Y_UNIT_TEST(TruncLevels3) {
- auto level0 = MakeIntrusive<TIssue>("level0");
- auto level1 = MakeIntrusive<TIssue>("level1");
- auto level2 = MakeIntrusive<TIssue>("level2");
- auto level3 = MakeIntrusive<TIssue>("level3");
- auto level40 = MakeIntrusive<TIssue>("level40");
- auto level41 = MakeIntrusive<TIssue>("level41");
- auto level50 = MakeIntrusive<TIssue>("level50");
-
- /*
- * * 0
- * |
- * * 1
- * |
- * * 2
- * |
- * * 3 --
- * | |
- * * 40 |
- * | |
- * * 50 * 41
- */
-
- level0->AddSubIssue(level1);
- level1->AddSubIssue(level2);
- level2->AddSubIssue(level3);
- level3->AddSubIssue(level40);
- level3->AddSubIssue(level41);
- level40->AddSubIssue(level50);
-
- {
- const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(1))}).ToString();
- const auto expected =
-R"___(<main>: Error: level0
- <main>: Error: level1
- <main>: Error: level2
- <main>: Error: (skipped levels)
- <main>: Error: level41
- <main>: Error: level50
-)___";
- UNIT_ASSERT_STRINGS_EQUAL(res, expected);
- }
-
- {
- const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString();
- const auto expected =
-R"___(<main>: Error: level0
- <main>: Error: level1
- <main>: Error: (skipped levels)
- <main>: Error: level3
- <main>: Error: level41
- <main>: Error: level40
- <main>: Error: level50
-)___";
- UNIT_ASSERT_STRINGS_EQUAL(res, expected);
- }
-
- {
- const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(3))}).ToString();
- const auto expected =
-R"___(<main>: Error: level0
- <main>: Error: (skipped levels)
- <main>: Error: level2
- <main>: Error: level3
- <main>: Error: level40
- <main>: Error: level50
- <main>: Error: level41
-)___";
- UNIT_ASSERT_STRINGS_EQUAL(res, expected);
- }
- }
-
- Y_UNIT_TEST(KeepSeverity) {
- const auto templ =
-R"___(<main>: {severity}: level0, code: 1
- <main>: {severity}: level1, code: 1
-)___";
- for (auto severity: {ESeverity::TSeverityIds_ESeverityId_S_INFO, ESeverity::TSeverityIds_ESeverityId_S_WARNING, ESeverity::TSeverityIds_ESeverityId_S_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL}) {
-
- auto level0 = MakeIntrusive<TIssue>(TIssue("level0").SetCode(1, severity));
- auto level1 = MakeIntrusive<TIssue>(TIssue("level1").SetCode(1, severity));
-
- level0->AddSubIssue(level1);
-
- const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(15).SetKeepTailLevels(3))}).ToString();
- const auto expected = SubstGlobalCopy<TString, TString>(templ, "{severity}", SeverityToString(severity));
- UNIT_ASSERT_STRINGS_EQUAL(res, expected);
- }
- }
-}
+#include "yql_issue_utils.h"
+#include "yql_issue.h"
+#include "yql_issue_id.h"
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/string/subst.h>
+
+using namespace NYql;
+
+Y_UNIT_TEST_SUITE(TIssueUtilsTest) {
+ Y_UNIT_TEST(TruncLevels1) {
+ auto level0 = MakeIntrusive<TIssue>("level0");
+ auto level1 = MakeIntrusive<TIssue>("level1");
+ auto level2 = MakeIntrusive<TIssue>("level2");
+ auto level30 = MakeIntrusive<TIssue>("level30");
+ auto level31 = MakeIntrusive<TIssue>("level31");
+ auto level40 = MakeIntrusive<TIssue>("level40");
+ auto level41 = MakeIntrusive<TIssue>("level41");
+ auto level51 = MakeIntrusive<TIssue>("level51");
+
+ /*
+ * * 0
+ * |
+ * * 1
+ * |
+ * * 2 --
+ * | |
+ * * 30 * 31
+ * | |
+ * * 40 * 41
+ * |
+ * * 51
+ */
+
+ level0->AddSubIssue(level1);
+ level1->AddSubIssue(level2);
+ level2->AddSubIssue(level30);
+ level2->AddSubIssue(level31);
+ level30->AddSubIssue(level40);
+ level31->AddSubIssue(level41);
+ level41->AddSubIssue(level51);
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: level1
+ <main>: Error: (skipped levels)
+ <main>: Error: level30
+ <main>: Error: level40
+ <main>: Error: level41
+ <main>: Error: level51
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(3).SetKeepTailLevels(1))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: level1
+ <main>: Error: (skipped levels)
+ <main>: Error: level40
+ <main>: Error: level51
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+ }
+
+ Y_UNIT_TEST(TruncLevels2) {
+ auto level0 = MakeIntrusive<TIssue>("level0");
+ auto level1 = MakeIntrusive<TIssue>("level1");
+ auto level2 = MakeIntrusive<TIssue>("level2");
+ auto level3 = MakeIntrusive<TIssue>("level3");
+ auto level40 = MakeIntrusive<TIssue>("level40");
+ auto level41 = MakeIntrusive<TIssue>("level41");
+
+ /*
+ * * 0
+ * |
+ * * 1
+ * |
+ * * 2
+ * |
+ * * 3 --
+ * | |
+ * * 40 * 41
+ */
+
+ level0->AddSubIssue(level1);
+ level1->AddSubIssue(level2);
+ level2->AddSubIssue(level3);
+ level3->AddSubIssue(level40);
+ level3->AddSubIssue(level41);
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: level1
+ <main>: Error: (skipped levels)
+ <main>: Error: level3
+ <main>: Error: level40
+ <main>: Error: level41
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+ }
+
+ Y_UNIT_TEST(TruncLevels3) {
+ auto level0 = MakeIntrusive<TIssue>("level0");
+ auto level1 = MakeIntrusive<TIssue>("level1");
+ auto level2 = MakeIntrusive<TIssue>("level2");
+ auto level3 = MakeIntrusive<TIssue>("level3");
+ auto level40 = MakeIntrusive<TIssue>("level40");
+ auto level41 = MakeIntrusive<TIssue>("level41");
+ auto level50 = MakeIntrusive<TIssue>("level50");
+
+ /*
+ * * 0
+ * |
+ * * 1
+ * |
+ * * 2
+ * |
+ * * 3 --
+ * | |
+ * * 40 |
+ * | |
+ * * 50 * 41
+ */
+
+ level0->AddSubIssue(level1);
+ level1->AddSubIssue(level2);
+ level2->AddSubIssue(level3);
+ level3->AddSubIssue(level40);
+ level3->AddSubIssue(level41);
+ level40->AddSubIssue(level50);
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(1))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: level1
+ <main>: Error: level2
+ <main>: Error: (skipped levels)
+ <main>: Error: level41
+ <main>: Error: level50
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(2))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: level1
+ <main>: Error: (skipped levels)
+ <main>: Error: level3
+ <main>: Error: level41
+ <main>: Error: level40
+ <main>: Error: level50
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+
+ {
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(4).SetKeepTailLevels(3))}).ToString();
+ const auto expected =
+R"___(<main>: Error: level0
+ <main>: Error: (skipped levels)
+ <main>: Error: level2
+ <main>: Error: level3
+ <main>: Error: level40
+ <main>: Error: level50
+ <main>: Error: level41
+)___";
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+ }
+
+ Y_UNIT_TEST(KeepSeverity) {
+ const auto templ =
+R"___(<main>: {severity}: level0, code: 1
+ <main>: {severity}: level1, code: 1
+)___";
+ for (auto severity: {ESeverity::TSeverityIds_ESeverityId_S_INFO, ESeverity::TSeverityIds_ESeverityId_S_WARNING, ESeverity::TSeverityIds_ESeverityId_S_ERROR, ESeverity::TSeverityIds_ESeverityId_S_FATAL}) {
+
+ auto level0 = MakeIntrusive<TIssue>(TIssue("level0").SetCode(1, severity));
+ auto level1 = MakeIntrusive<TIssue>(TIssue("level1").SetCode(1, severity));
+
+ level0->AddSubIssue(level1);
+
+ const auto res = TIssues({TruncateIssueLevels(*level0, TTruncateIssueOpts().SetMaxLevels(15).SetKeepTailLevels(3))}).ToString();
+ const auto expected = SubstGlobalCopy<TString, TString>(templ, "{severity}", SeverityToString(severity));
+ UNIT_ASSERT_STRINGS_EQUAL(res, expected);
+ }
+ }
+}
diff --git a/ydb/library/yql/public/udf/udf_helpers.cpp b/ydb/library/yql/public/udf/udf_helpers.cpp
index 260ff3e544..0b0e5522ea 100644
--- a/ydb/library/yql/public/udf/udf_helpers.cpp
+++ b/ydb/library/yql/public/udf/udf_helpers.cpp
@@ -1,27 +1,27 @@
-#include "udf_helpers.h"
-
+#include "udf_helpers.h"
+
#include <library/cpp/resource/resource.h>
-
-#include <util/generic/hash.h>
-#include <util/generic/singleton.h>
-
+
+#include <util/generic/hash.h>
+#include <util/generic/singleton.h>
+
namespace NYql {
-namespace NUdf {
-
-struct TLoadedResources {
- TString Get(TStringBuf resourceId) {
- if (auto p = Resources.FindPtr(resourceId)) {
- return *p;
- }
- return Resources.emplace(resourceId, NResource::Find(resourceId)).first->second;
- }
-
- THashMap<TString, TString> Resources;
-};
-
-TString LoadResourceOnce(TStringBuf resourceId) {
- return Singleton<TLoadedResources>()->Get(resourceId);
-}
-
-}
-}
+namespace NUdf {
+
+struct TLoadedResources {
+ TString Get(TStringBuf resourceId) {
+ if (auto p = Resources.FindPtr(resourceId)) {
+ return *p;
+ }
+ return Resources.emplace(resourceId, NResource::Find(resourceId)).first->second;
+ }
+
+ THashMap<TString, TString> Resources;
+};
+
+TString LoadResourceOnce(TStringBuf resourceId) {
+ return Singleton<TLoadedResources>()->Get(resourceId);
+}
+
+}
+}
diff --git a/ydb/library/yql/public/udf/udf_helpers.h b/ydb/library/yql/public/udf/udf_helpers.h
index 2be9443559..b43f2346f9 100644
--- a/ydb/library/yql/public/udf/udf_helpers.h
+++ b/ydb/library/yql/public/udf/udf_helpers.h
@@ -6,11 +6,11 @@
#include "udf_terminator.h"
#include "udf_type_builder.h"
#include "udf_type_inspection.h"
-#include "udf_version.h"
+#include "udf_version.h"
#include <util/generic/yexception.h>
-#include <util/generic/string.h>
-#include <util/generic/strbuf.h>
+#include <util/generic/string.h>
+#include <util/generic/strbuf.h>
#include <util/string/builder.h>
@@ -25,20 +25,20 @@ namespace NUdf {
return {};
#endif
}
-
- TString LoadResourceOnce(TStringBuf resourceId);
-
- inline void SetIRImplementation(IFunctionTypeInfoBuilder& builder, TStringBuf resourceId, TStringBuf functionName) {
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 16)
- if (functionName) {
- builder.IRImplementation(LoadResourceOnce(resourceId), resourceId, functionName);
- }
-#else
- Y_UNUSED(builder);
- Y_UNUSED(resourceId);
- Y_UNUSED(functionName);
-#endif
- }
+
+ TString LoadResourceOnce(TStringBuf resourceId);
+
+ inline void SetIRImplementation(IFunctionTypeInfoBuilder& builder, TStringBuf resourceId, TStringBuf functionName) {
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 16)
+ if (functionName) {
+ builder.IRImplementation(LoadResourceOnce(resourceId), resourceId, functionName);
+ }
+#else
+ Y_UNUSED(builder);
+ Y_UNUSED(resourceId);
+ Y_UNUSED(functionName);
+#endif
+ }
}
}
@@ -79,7 +79,7 @@ namespace NUdf {
if (Name() == name) { \
typeBody if (!typesOnly) { \
create_impl \
- SetIRImplementation(builder, irResourceId, irFunctionName); \
+ SetIRImplementation(builder, irResourceId, irFunctionName); \
} \
return true; \
} else { \
@@ -97,9 +97,9 @@ namespace NUdf {
#define UDF_IMPL(udfName, typeBody, members, init, irResourceId, irFunctionName) \
UDF_IMPL_EX(udfName, typeBody, members, init, irResourceId, irFunctionName, builder.Implementation(new udfName(builder));)
-#define UDF(udfName, typeBody) UDF_IMPL(udfName, typeBody, ;, ;, "", "")
+#define UDF(udfName, typeBody) UDF_IMPL(udfName, typeBody, ;, ;, "", "")
-#define UDF_RUN_IMPL(udfName, typeBody, members, init, irResourceId, irFunctionName) \
+#define UDF_RUN_IMPL(udfName, typeBody, members, init, irResourceId, irFunctionName) \
struct udfName##Members { \
members \
}; \
@@ -160,7 +160,7 @@ namespace NUdf {
if (Name() == name) { \
typeBody if (!typesOnly) { \
builder.Implementation(new udfName(builder)); \
- SetIRImplementation(builder, irResourceId, irFunctionName); \
+ SetIRImplementation(builder, irResourceId, irFunctionName); \
} \
return true; \
} else { \
@@ -174,14 +174,14 @@ namespace NUdf {
const ::NYql::NUdf::IValueBuilder* valueBuilder, \
const ::NYql::NUdf::TUnboxedValuePod* args) const
-#define UDF_RUN(udfName, typeBody) UDF_RUN_IMPL(udfName, typeBody, ;, ;, "", "")
+#define UDF_RUN(udfName, typeBody) UDF_RUN_IMPL(udfName, typeBody, ;, ;, "", "")
#define SIMPLE_UDF(udfName, signature) \
UDF(udfName, builder.SimpleSignature<signature>();)
-#define SIMPLE_UDF_WITH_IR(udfName, signature, irResourceId, irFunctionName) \
- UDF_IMPL(udfName, builder.SimpleSignature<signature>();, ;, ;, irResourceId, irFunctionName)
-
+#define SIMPLE_UDF_WITH_IR(udfName, signature, irResourceId, irFunctionName) \
+ UDF_IMPL(udfName, builder.SimpleSignature<signature>();, ;, ;, irResourceId, irFunctionName)
+
#define SIMPLE_UDF_WITH_CREATE_IMPL(udfName, signature, create_impl) \
UDF_IMPL_EX(udfName, builder.SimpleSignature<signature>();, ;, ;, "", "", create_impl)
diff --git a/ydb/library/yql/public/udf/udf_type_builder.h b/ydb/library/yql/public/udf/udf_type_builder.h
index 8bfc73a449..a77c6f2b95 100644
--- a/ydb/library/yql/public/udf/udf_type_builder.h
+++ b/ydb/library/yql/public/udf/udf_type_builder.h
@@ -532,17 +532,17 @@ public:
virtual TType* Decimal(ui8 precision, ui8 scale) const = 0;
};
#endif
-
-#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 16)
+
+#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 16)
class IFunctionTypeInfoBuilder7: public IFunctionTypeInfoBuilder6 {
public:
virtual IFunctionTypeInfoBuilder7& IRImplementationImpl(
- const TStringRef& moduleIR,
- const TStringRef& moduleIRUniqId,
- const TStringRef& functionName
- ) = 0;
+ const TStringRef& moduleIR,
+ const TStringRef& moduleIRUniqId,
+ const TStringRef& functionName
+ ) = 0;
};
-#endif
+#endif
#if UDF_ABI_COMPATIBILITY_VERSION_CURRENT >= UDF_ABI_COMPATIBILITY_VERSION(2, 18)
class IFunctionTypeInfoBuilder8: public IFunctionTypeInfoBuilder7 {
@@ -871,7 +871,7 @@ struct TTypeBuilderHelper<TTuple<TArgs...>> {
};
template <typename... TArgs>
-struct TTypeBuilderHelper<NUdf::TVariant<TArgs...>> {
+struct TTypeBuilderHelper<NUdf::TVariant<TArgs...>> {
static TType* Build(const IFunctionTypeInfoBuilder& builder) {
auto tupleBuilder = builder.Tuple(sizeof...(TArgs));
TTupleHelper<TArgs...>::Add(*tupleBuilder, builder);
diff --git a/ydb/library/yql/public/udf/udf_type_ops.h b/ydb/library/yql/public/udf/udf_type_ops.h
index 4b2446c8fd..37f1a0b73f 100644
--- a/ydb/library/yql/public/udf/udf_type_ops.h
+++ b/ydb/library/yql/public/udf/udf_type_ops.h
@@ -16,23 +16,23 @@ inline THashType GetValueHash(const TUnboxedValuePod& value);
inline THashType GetValueHash(EDataSlot type, const TUnboxedValuePod& value);
template <EDataSlot Type>
-inline int CompareValues(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
-inline int CompareValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
+inline int CompareValues(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
+inline int CompareValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
template <EDataSlot Type>
-inline bool EquateValues(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
-inline bool EquateValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
+inline bool EquateValues(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
+inline bool EquateValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs);
template <EDataSlot Type>
struct TUnboxedValueHash {
- std::size_t operator()(const TUnboxedValuePod& value) const {
+ std::size_t operator()(const TUnboxedValuePod& value) const {
return static_cast<std::size_t>(GetValueHash<Type>(value));
}
};
template <EDataSlot Type>
struct TUnboxedValueEquals {
- bool operator()(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) const {
+ bool operator()(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) const {
return EquateValues<Type>(lhs, rhs);
}
};
@@ -207,14 +207,14 @@ inline THashType GetValueHash(EDataSlot type, const TUnboxedValuePod& value) {
// compare
template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
-inline int CompareIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
const auto x = lhs.Get<T>();
const auto y = rhs.Get<T>();
return (x == y) ? 0 : (x < y ? -1 : 1);
}
template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
-inline int CompareFloats(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareFloats(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
const auto x = lhs.Get<T>();
const auto y = rhs.Get<T>();
if (std::isunordered(x, y)) {
@@ -225,14 +225,14 @@ inline int CompareFloats(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rh
return (x == y) ? 0 : (x < y ? -1 : 1);
}
-inline int CompareStrings(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareStrings(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
const TStringBuf lhsBuf = lhs.AsStringRef();
const TStringBuf rhsBuf = rhs.AsStringRef();
return lhsBuf.compare(rhsBuf);
}
template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
-inline int CompareTzIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareTzIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
const auto x = lhs.Get<T>();
const auto y = rhs.Get<T>();
if (x < y) {
@@ -249,84 +249,84 @@ inline int CompareTzIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod
}
template <>
-inline int CompareValues<EDataSlot::Bool>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Bool>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
const auto x = lhs.Get<bool>();
const auto y = rhs.Get<bool>();
return x == y ? 0 : (!x ? -1 : 1);
}
template <>
-inline int CompareValues<EDataSlot::Int8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Int8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<i8>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Uint8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Uint8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<ui8>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Int16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Int16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<i16>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Uint16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Uint16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<ui16>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Int32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Int32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<i32>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Uint32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Uint32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<ui32>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Int64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Int64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<i64>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Uint64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Uint64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<ui64>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Float>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Float>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareFloats<float>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Double>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Double>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareFloats<double>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::String>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::String>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareStrings(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Utf8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Utf8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareStrings(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Uuid>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Uuid>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareStrings(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Yson>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
+inline int CompareValues<EDataSlot::Yson>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
Y_FAIL("Yson isn't comparable.");
}
template <>
-inline int CompareValues<EDataSlot::Json>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
+inline int CompareValues<EDataSlot::Json>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
Y_FAIL("Json isn't comparable.");
}
@@ -336,42 +336,42 @@ inline int CompareValues<EDataSlot::JsonDocument>(const TUnboxedValuePod&, const
}
template <>
-inline int CompareValues<EDataSlot::Date>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Date>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<ui16>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Datetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Datetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<ui32>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Timestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Timestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<ui64>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Interval>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Interval>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareIntegers<i64>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::TzDate>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::TzDate>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareTzIntegers<ui16>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::TzDatetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::TzDatetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareTzIntegers<ui32>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::TzTimestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::TzTimestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return CompareTzIntegers<ui64>(lhs, rhs);
}
template <>
-inline int CompareValues<EDataSlot::Decimal>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues<EDataSlot::Decimal>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
const auto x = lhs.GetInt128();
const auto y = rhs.GetInt128();
return x == y ? 0 : (x < y ? -1 : 1);
@@ -382,7 +382,7 @@ inline int CompareValues<EDataSlot::DyNumber>(const TUnboxedValuePod& lhs, const
return CompareStrings(lhs, rhs);
}
-inline int CompareValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline int CompareValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
#define COMPARE_TYPE(slot, ...) \
case EDataSlot::slot: \
return CompareValues<EDataSlot::slot>(lhs, rhs);
@@ -398,105 +398,105 @@ inline int CompareValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnb
// equate
template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
-inline bool EquateIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return lhs.Get<T>() == rhs.Get<T>();
}
template <typename T, std::enable_if_t<std::is_floating_point<T>::value>* = nullptr>
-inline bool EquateFloats(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateFloats(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
const auto x = lhs.Get<T>();
const auto y = rhs.Get<T>();
return std::isunordered(x, y) ? std::isnan(x) == std::isnan(y) : x == y;
}
-inline bool EquateStrings(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateStrings(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
const auto& lhsBuf = lhs.AsStringRef();
const auto& rhsBuf = rhs.AsStringRef();
return lhsBuf == rhsBuf;
}
template <typename T, std::enable_if_t<std::is_integral<T>::value>* = nullptr>
-inline bool EquateTzIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateTzIntegers(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return lhs.Get<T>() == rhs.Get<T>() && lhs.GetTimezoneId() == rhs.GetTimezoneId();
}
template <>
-inline bool EquateValues<EDataSlot::Bool>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Bool>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<bool>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Int8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Int8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<i8>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Uint8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Uint8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<ui8>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Int16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Int16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<i16>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Uint16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Uint16>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<ui16>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Int32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Int32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<i32>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Uint32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Uint32>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<ui32>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Int64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Int64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<i64>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Uint64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Uint64>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<ui64>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Float>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Float>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateFloats<float>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Double>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Double>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateFloats<double>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::String>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::String>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateStrings(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Utf8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Utf8>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateStrings(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Uuid>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Uuid>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateStrings(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Yson>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
+inline bool EquateValues<EDataSlot::Yson>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
Y_FAIL("Yson isn't comparable.");
}
template <>
-inline bool EquateValues<EDataSlot::Json>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
+inline bool EquateValues<EDataSlot::Json>(const TUnboxedValuePod&, const TUnboxedValuePod&) {
Y_FAIL("Json isn't comparable.");
}
@@ -506,42 +506,42 @@ inline bool EquateValues<EDataSlot::JsonDocument>(const TUnboxedValuePod&, const
}
template <>
-inline bool EquateValues<EDataSlot::Date>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Date>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<ui16>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Datetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Datetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<ui32>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Timestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Timestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<ui64>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Interval>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Interval>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateIntegers<i64>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::TzDate>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::TzDate>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateTzIntegers<ui16>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::TzDatetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::TzDatetime>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateTzIntegers<ui32>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::TzTimestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::TzTimestamp>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return EquateTzIntegers<ui64>(lhs, rhs);
}
template <>
-inline bool EquateValues<EDataSlot::Decimal>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues<EDataSlot::Decimal>(const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
return lhs.GetInt128() == rhs.GetInt128();
}
@@ -550,7 +550,7 @@ inline bool EquateValues<EDataSlot::DyNumber>(const TUnboxedValuePod& lhs, const
return EquateStrings(lhs, rhs);
}
-inline bool EquateValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
+inline bool EquateValues(EDataSlot type, const TUnboxedValuePod& lhs, const TUnboxedValuePod& rhs) {
#define EQUATE_TYPE(slot, ...) \
case EDataSlot::slot: \
return EquateValues<EDataSlot::slot>(lhs, rhs);
diff --git a/ydb/library/yql/public/udf/udf_value.h b/ydb/library/yql/public/udf/udf_value.h
index 705ac41133..992303ced0 100644
--- a/ydb/library/yql/public/udf/udf_value.h
+++ b/ydb/library/yql/public/udf/udf_value.h
@@ -10,7 +10,7 @@
#include <util/system/yassert.h> // FAIL, VERIFY_DEBUG
#include <util/generic/utility.h> // Min, Max
#include <util/generic/yexception.h> // Y_ENSURE
-#include <util/system/compiler.h> // Y_FORCE_INLINE
+#include <util/system/compiler.h> // Y_FORCE_INLINE
#include <algorithm>
#include <type_traits>
diff --git a/ydb/library/yql/public/udf/udf_value_inl.h b/ydb/library/yql/public/udf/udf_value_inl.h
index eb2db28e18..841aa11479 100644
--- a/ydb/library/yql/public/udf/udf_value_inl.h
+++ b/ydb/library/yql/public/udf/udf_value_inl.h
@@ -5,22 +5,22 @@
#endif // INCLUDE_UDF_VALUE_INL_H
#ifdef LLVM_BC
-
-#define UDF_VERIFY(expr, ...) \
- do { \
- if (false) { \
- bool __xxx = static_cast<bool>(expr); \
- Y_UNUSED(__xxx); \
- } \
- } while (false)
-
-#define UDF_ALWAYS_INLINE __attribute__((always_inline))
-
+
+#define UDF_VERIFY(expr, ...) \
+ do { \
+ if (false) { \
+ bool __xxx = static_cast<bool>(expr); \
+ Y_UNUSED(__xxx); \
+ } \
+ } while (false)
+
+#define UDF_ALWAYS_INLINE __attribute__((always_inline))
+
#else
-
+
#define UDF_VERIFY Y_VERIFY_DEBUG
-#define UDF_ALWAYS_INLINE Y_FORCE_INLINE
-
+#define UDF_ALWAYS_INLINE Y_FORCE_INLINE
+
#endif
//////////////////////////////////////////////////////////////////////////////
diff --git a/ydb/library/yql/sql/settings/translation_settings.h b/ydb/library/yql/sql/settings/translation_settings.h
index 0b9194a249..4fcaeaadc4 100644
--- a/ydb/library/yql/sql/settings/translation_settings.h
+++ b/ydb/library/yql/sql/settings/translation_settings.h
@@ -19,11 +19,11 @@ namespace NSQLTranslation {
LIMITED_VIEW = 1,
LIBRARY = 2,
SUBQUERY = 3,
- DISCOVERY = 4,
+ DISCOVERY = 4,
};
inline bool IsQueryMode(NSQLTranslation::ESqlMode mode) {
- return mode == NSQLTranslation::ESqlMode::QUERY || mode == NSQLTranslation::ESqlMode::DISCOVERY;
+ return mode == NSQLTranslation::ESqlMode::QUERY || mode == NSQLTranslation::ESqlMode::DISCOVERY;
}
using TIncrementMonCounterFunction = std::function<void(const TString&, const TString&)>;
diff --git a/ydb/library/yql/sql/v0/SQL.g b/ydb/library/yql/sql/v0/SQL.g
index 3b616b9c9f..ce4d52ed54 100644
--- a/ydb/library/yql/sql/v0/SQL.g
+++ b/ydb/library/yql/sql/v0/SQL.g
@@ -200,12 +200,12 @@ select_kind_partial: select_kind
select_kind: (DISCARD)? (process_core | reduce_core | select_core) (INTO RESULT pure_column_or_named)?;
process_core:
- PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING call_expr (AS id_or_string)?
+ PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING call_expr (AS id_or_string)?
(WHERE expr)? (HAVING expr)?)?
;
reduce_core:
- REDUCE named_single_source (COMMA named_single_source)* (PRESORT sort_specification_list)?
+ REDUCE named_single_source (COMMA named_single_source)* (PRESORT sort_specification_list)?
ON column_list USING ALL? call_expr (AS id_or_string)?
(WHERE expr)? (HAVING expr)?
;
diff --git a/ydb/library/yql/sql/v0/builtin.cpp b/ydb/library/yql/sql/v0/builtin.cpp
index a0945f9d05..cdd30cfb84 100644
--- a/ydb/library/yql/sql/v0/builtin.cpp
+++ b/ydb/library/yql/sql/v0/builtin.cpp
@@ -358,7 +358,7 @@ public:
}
if (Args.empty()) {
- Args.push_back(Y("TablePath", Y("DependsOn", "row")));
+ Args.push_back(Y("TablePath", Y("DependsOn", "row")));
}
if (Args.size() == 2) {
diff --git a/ydb/library/yql/sql/v0/context.cpp b/ydb/library/yql/sql/v0/context.cpp
index a3a9013f2b..2d374e234e 100644
--- a/ydb/library/yql/sql/v0/context.cpp
+++ b/ydb/library/yql/sql/v0/context.cpp
@@ -36,12 +36,12 @@ TNodePtr AddTablePathPrefix(TContext &ctx, TStringBuf prefixPath, const TDeferre
return BuildQuotedAtom(path.Build()->GetPos(), prefixPathSplit.AppendMany(pathSplit.begin(), pathSplit.end()).Reconstruct());
}
-typedef bool TContext::*TPragmaField;
-
-THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = {
+typedef bool TContext::*TPragmaField;
+
+THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = {
{"PullUpFlatMapOverJoin", &TContext::PragmaPullUpFlatMapOverJoin},
-};
-
+};
+
} // namespace
TContext::TContext(const NSQLTranslation::TTranslationSettings& settings,
@@ -59,18 +59,18 @@ TContext::TContext(const NSQLTranslation::TTranslationSettings& settings,
{
Position.File = settings.File;
- for (auto& flag: settings.Flags) {
- bool value = true;
- TStringBuf key = flag;
- auto ptr = CTX_PRAGMA_FIELDS.FindPtr(key);
- if (!ptr && key.SkipPrefix("Disable")) {
- value = false;
- ptr = CTX_PRAGMA_FIELDS.FindPtr(key);
- }
+ for (auto& flag: settings.Flags) {
+ bool value = true;
+ TStringBuf key = flag;
+ auto ptr = CTX_PRAGMA_FIELDS.FindPtr(key);
+ if (!ptr && key.SkipPrefix("Disable")) {
+ value = false;
+ ptr = CTX_PRAGMA_FIELDS.FindPtr(key);
+ }
if (ptr) {
this->*(*ptr) = value;
}
- }
+ }
}
TContext::~TContext()
@@ -147,10 +147,10 @@ IOutputStream& TContext::MakeIssue(ESeverity severity, TIssueCode code, NYql::TP
bool TContext::SetPathPrefix(const TString& value, TMaybe<TString> arg) {
if (arg.Defined()) {
- if (*arg == YtProviderName
- || *arg == KikimrProviderName
- || *arg == RtmrProviderName
- )
+ if (*arg == YtProviderName
+ || *arg == KikimrProviderName
+ || *arg == RtmrProviderName
+ )
{
ProviderPathPrefixes[*arg] = value;
return true;
@@ -309,34 +309,34 @@ TString TContext::AddSimpleUdf(const TString& udf) {
return name;
}
-TString TContext::GetServiceName(const ISource& source) const {
- TTableList tableList;
- source.GetInputTables(tableList);
-
- TSet<TString> clusters;
- for (auto& it: tableList) {
- if (auto provider = GetClusterProvider(it.Cluster)) {
- return *provider;
- }
- }
-
- for (auto& cluster: UsedClusters) {
- if (auto provider = GetClusterProvider(cluster)) {
- return *provider;
- }
- }
-
- return CurrCluster.empty() ? TString() : GetClusterProvider(CurrCluster).GetOrElse(TString());
-}
-
-bool TContext::UseUnordered(const ISource& source) const {
- return YtProviderName == to_lower(GetServiceName(source));
-}
-
-bool TContext::UseUnordered(const TTableRef& table) const {
- return YtProviderName == to_lower(GetClusterProvider(table.Cluster).GetOrElse(TString()));
-}
-
+TString TContext::GetServiceName(const ISource& source) const {
+ TTableList tableList;
+ source.GetInputTables(tableList);
+
+ TSet<TString> clusters;
+ for (auto& it: tableList) {
+ if (auto provider = GetClusterProvider(it.Cluster)) {
+ return *provider;
+ }
+ }
+
+ for (auto& cluster: UsedClusters) {
+ if (auto provider = GetClusterProvider(cluster)) {
+ return *provider;
+ }
+ }
+
+ return CurrCluster.empty() ? TString() : GetClusterProvider(CurrCluster).GetOrElse(TString());
+}
+
+bool TContext::UseUnordered(const ISource& source) const {
+ return YtProviderName == to_lower(GetServiceName(source));
+}
+
+bool TContext::UseUnordered(const TTableRef& table) const {
+ return YtProviderName == to_lower(GetClusterProvider(table.Cluster).GetOrElse(TString()));
+}
+
TTranslation::TTranslation(TContext& ctx)
: Ctx(ctx)
{
diff --git a/ydb/library/yql/sql/v0/context.h b/ydb/library/yql/sql/v0/context.h
index 45f40fae61..e7f17082c0 100644
--- a/ydb/library/yql/sql/v0/context.h
+++ b/ydb/library/yql/sql/v0/context.h
@@ -80,10 +80,10 @@ namespace NSQLTranslationV0 {
return provider;
}
- TString GetServiceName(const ISource& source) const;
- bool UseUnordered(const ISource& source) const;
- bool UseUnordered(const TTableRef& table) const;
-
+ TString GetServiceName(const ISource& source) const;
+ bool UseUnordered(const ISource& source) const;
+ bool UseUnordered(const TTableRef& table) const;
+
bool SetPathPrefix(const TString& value, TMaybe<TString> arg = TMaybe<TString>());
TNodePtr GetPrefixedPath(const TString& cluster, const TDeferredAtom& path);
@@ -144,7 +144,7 @@ namespace NSQLTranslationV0 {
bool PragmaInferSchema = false;
bool PragmaAutoCommit = false;
bool SimpleColumns = false;
- bool PragmaDirectRead = false;
+ bool PragmaDirectRead = false;
bool PragmaYsonAutoConvert = false;
bool PragmaYsonStrict = false;
bool PragmaClassicDivision = true;
diff --git a/ydb/library/yql/sql/v0/insert.cpp b/ydb/library/yql/sql/v0/insert.cpp
index 1d8aec7134..2ffec32806 100644
--- a/ydb/library/yql/sql/v0/insert.cpp
+++ b/ydb/library/yql/sql/v0/insert.cpp
@@ -45,8 +45,8 @@ public:
return false;
}
- TNodePtr BuildFilter(TContext& ctx, const TString& label, const TNodePtr& groundNode) override {
- Y_UNUSED(ctx);
+ TNodePtr BuildFilter(TContext& ctx, const TString& label, const TNodePtr& groundNode) override {
+ Y_UNUSED(ctx);
Y_UNUSED(label);
Y_UNUSED(groundNode);
return nullptr;
@@ -241,10 +241,10 @@ public:
return new TModifyBySource(Pos, OperationHumanName, ColumnsHint, Source->CloneSource());
}
- bool IsOrdered() const final {
- return Source->IsOrdered();
- }
-
+ bool IsOrdered() const final {
+ return Source->IsOrdered();
+ }
+
private:
TString OperationHumanName;
TSourcePtr Source;
@@ -311,7 +311,7 @@ public:
options = L(options, Q(Y(Q("filter"), TableSource->BuildFilterLambda(ctx.GroundBlockShortcuts(Pos)))));
}
- bool unordered = false;
+ bool unordered = false;
ctx.PushBlockShortcuts();
if (Values) {
if (!Values->Init(ctx, TableSource.Get())) {
@@ -324,7 +324,7 @@ public:
if (!values) {
return false;
}
- unordered = !Values->IsOrdered();
+ unordered = !Values->IsOrdered();
}
TNodePtr node(BuildInputTables(Pos, tableList, false));
@@ -346,9 +346,9 @@ public:
node = ctx.GroundBlockShortcuts(Pos, node);
if (values) {
node = L(node, Y("let", "values", values));
- if (unordered && ctx.UseUnordered(Table)) {
- node = L(node, Y("let", "values", Y("Unordered", "values")));
- }
+ if (unordered && ctx.UseUnordered(Table)) {
+ node = L(node, Y("let", "values", Y("Unordered", "values")));
+ }
} else {
node = L(node, Y("let", "values", Y("Void")));
}
diff --git a/ydb/library/yql/sql/v0/join.cpp b/ydb/library/yql/sql/v0/join.cpp
index 78f5affb49..7dfefc870d 100644
--- a/ydb/library/yql/sql/v0/join.cpp
+++ b/ydb/library/yql/sql/v0/join.cpp
@@ -477,14 +477,14 @@ public:
}
TNodePtr equiJoin(Y("EquiJoin"));
- bool ordered = false;
+ bool ordered = false;
for (auto& source: Sources) {
auto sourceNode = source->Build(ctx);
if (!sourceNode) {
return nullptr;
}
- const bool useOrderedForSource = ctx.UseUnordered(*source);
- ordered = ordered || useOrderedForSource;
+ const bool useOrderedForSource = ctx.UseUnordered(*source);
+ ordered = ordered || useOrderedForSource;
if (source->IsFlattenByColumns() || source->IsFlattenColumns()) {
auto flatten = source->IsFlattenByColumns() ?
source->BuildFlattenByColumns("row") :
@@ -494,7 +494,7 @@ public:
return nullptr;
}
auto block = Y(Y("let", "flatten", sourceNode));
- block = L(block, Y("let", "flatten", Y(useOrderedForSource ? "OrderedFlatMap" : "FlatMap", "flatten", BuildLambda(Pos, Y("row"), flatten, "res"))));
+ block = L(block, Y("let", "flatten", Y(useOrderedForSource ? "OrderedFlatMap" : "FlatMap", "flatten", BuildLambda(Pos, Y("row"), flatten, "res"))));
sourceNode = Y("block", Q(L(block, Y("return", "flatten"))));
}
TNodePtr extraMembers;
@@ -511,7 +511,7 @@ public:
);
}
if (extraMembers) {
- sourceNode = Y(useOrderedForSource ? "OrderedMap" : "Map", sourceNode, BuildLambda(Pos, Y("row"), extraMembers, "row"));
+ sourceNode = Y(useOrderedForSource ? "OrderedMap" : "Map", sourceNode, BuildLambda(Pos, Y("row"), extraMembers, "row"));
}
if (ctx.EnableSystemColumns && source->IsTableSource()) {
sourceNode = Y("RemoveSystemMembers", sourceNode);
@@ -531,7 +531,7 @@ public:
auto options = Y();
equiJoin = L(equiJoin, joinTree, Q(options));
if (removeMembers) {
- equiJoin = Y(ordered ? "OrderedMap" : "Map", equiJoin, BuildLambda(Pos, Y("row"), removeMembers, "row"));
+ equiJoin = Y(ordered ? "OrderedMap" : "Map", equiJoin, BuildLambda(Pos, Y("row"), removeMembers, "row"));
}
return equiJoin;
}
diff --git a/ydb/library/yql/sql/v0/node.cpp b/ydb/library/yql/sql/v0/node.cpp
index 57f0c6a561..5bbf0097e7 100644
--- a/ydb/library/yql/sql/v0/node.cpp
+++ b/ydb/library/yql/sql/v0/node.cpp
@@ -585,11 +585,11 @@ bool TCallDirectRow::DoInit(TContext& ctx, ISource* src) {
ctx.Error(Pos) << "Failed to use function: " << OpName << " with aggregation, join, flatten by or window functions";
return false;
}
- if (!TCallNode::DoInit(ctx, src)) {
- return false;
- }
- Nodes.push_back(Y("DependsOn", "row"));
- return true;
+ if (!TCallNode::DoInit(ctx, src)) {
+ return false;
+ }
+ Nodes.push_back(Y("DependsOn", "row"));
+ return true;
}
void TCallDirectRow::DoUpdateState() const {
@@ -1325,10 +1325,10 @@ bool ISource::IsStream() const {
return false;
}
-bool ISource::IsOrdered() const {
- return false;
-}
-
+bool ISource::IsOrdered() const {
+ return false;
+}
+
TWriteSettings ISource::GetWriteSettings() const {
return {};
}
@@ -1353,8 +1353,8 @@ bool ISource::CalculateGroupingHint(TContext& ctx, const TVector<TString>& colum
return false;
}
-TNodePtr ISource::BuildFilter(TContext& ctx, const TString& label, const TNodePtr& groundNode) {
- return Filters.empty() ? nullptr : Y(ctx.UseUnordered(*this) ? "OrderedFilter" : "Filter", label, BuildFilterLambda(groundNode));
+TNodePtr ISource::BuildFilter(TContext& ctx, const TString& label, const TNodePtr& groundNode) {
+ return Filters.empty() ? nullptr : Y(ctx.UseUnordered(*this) ? "OrderedFilter" : "Filter", label, BuildFilterLambda(groundNode));
}
TNodePtr ISource::BuildFilterLambda(const TNodePtr& groundNode) {
@@ -1419,7 +1419,7 @@ TNodePtr ISource::BuildPrewindowMap(TContext& ctx, const TNodePtr& groundNode) {
feed = Y("AddMember", feed, Q(name), GroundWithExpr(groundNode, exprNode));
}
}
- return Y(ctx.UseUnordered(*this) ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), Y("AsList", feed)));
+ return Y(ctx.UseUnordered(*this) ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), Y("AsList", feed)));
}
TNodePtr ISource::BuildAggregation(const TString& label) {
@@ -2849,43 +2849,43 @@ bool TryMakeClusterAndTableFromExpression(TNodePtr node, TString& cluster, TDefe
return true;
}
-class TTupleResultNode: public INode {
-public:
- TTupleResultNode(TNodePtr&& tuple, int ensureTupleSize)
- : INode(tuple->GetPos())
- , Node(std::move(tuple))
- , EnsureTupleSize(ensureTupleSize)
- {
- }
-
- bool DoInit(TContext& ctx, ISource* src) override {
- ctx.PushBlockShortcuts();
- if (!Node->Init(ctx, src)) {
- return false;
- }
-
- Node = ctx.GroundBlockShortcutsForExpr(Node);
- Node = Y("EnsureTupleSize", Node, Q(ToString(EnsureTupleSize)));
-
- return true;
- }
-
- TAstNode* Translate(TContext& ctx) const override {
- return Node->Translate(ctx);
- }
-
- TPtr DoClone() const final {
- return {};
- }
-
-protected:
- TNodePtr Node;
- const int EnsureTupleSize;
-};
-
-TNodePtr BuildTupleResult(TNodePtr tuple, int ensureTupleSize) {
- return new TTupleResultNode(std::move(tuple), ensureTupleSize);
-}
-
-
+class TTupleResultNode: public INode {
+public:
+ TTupleResultNode(TNodePtr&& tuple, int ensureTupleSize)
+ : INode(tuple->GetPos())
+ , Node(std::move(tuple))
+ , EnsureTupleSize(ensureTupleSize)
+ {
+ }
+
+ bool DoInit(TContext& ctx, ISource* src) override {
+ ctx.PushBlockShortcuts();
+ if (!Node->Init(ctx, src)) {
+ return false;
+ }
+
+ Node = ctx.GroundBlockShortcutsForExpr(Node);
+ Node = Y("EnsureTupleSize", Node, Q(ToString(EnsureTupleSize)));
+
+ return true;
+ }
+
+ TAstNode* Translate(TContext& ctx) const override {
+ return Node->Translate(ctx);
+ }
+
+ TPtr DoClone() const final {
+ return {};
+ }
+
+protected:
+ TNodePtr Node;
+ const int EnsureTupleSize;
+};
+
+TNodePtr BuildTupleResult(TNodePtr tuple, int ensureTupleSize) {
+ return new TTupleResultNode(std::move(tuple), ensureTupleSize);
+}
+
+
} // namespace NSQLTranslationV0
diff --git a/ydb/library/yql/sql/v0/node.h b/ydb/library/yql/sql/v0/node.h
index 4c7cfc273b..11989b7f8a 100644
--- a/ydb/library/yql/sql/v0/node.h
+++ b/ydb/library/yql/sql/v0/node.h
@@ -726,11 +726,11 @@ namespace NSQLTranslationV0 {
virtual bool IsCalcOverWindow() const;
virtual bool IsOverWindowSource() const;
virtual bool IsStream() const;
- virtual bool IsOrdered() const;
+ virtual bool IsOrdered() const;
virtual TWriteSettings GetWriteSettings() const;
virtual bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed);
virtual bool CalculateGroupingHint(TContext& ctx, const TVector<TString>& columns, ui64& hint) const;
- virtual TNodePtr BuildFilter(TContext& ctx, const TString& label, const TNodePtr& groundNode);
+ virtual TNodePtr BuildFilter(TContext& ctx, const TString& label, const TNodePtr& groundNode);
virtual TNodePtr BuildFilterLambda(const TNodePtr& groundNode);
virtual TNodePtr BuildFlattenByColumns(const TString& label);
virtual TNodePtr BuildFlattenColumns(const TString& label);
@@ -902,7 +902,7 @@ namespace NSQLTranslationV0 {
TNodePtr BuildShortcutNode(const TNodePtr& node, const TString& baseName);
TNodePtr BuildDoCall(TPosition pos, const TNodePtr& node);
- TNodePtr BuildTupleResult(TNodePtr tuple, int ensureTupleSize);
+ TNodePtr BuildTupleResult(TNodePtr tuple, int ensureTupleSize);
// Implemented in aggregation.cpp
TAggregationPtr BuildFactoryAggregation(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode, bool multi = false);
@@ -944,9 +944,9 @@ namespace NSQLTranslationV0 {
// Implemented in select.cpp
TNodePtr BuildSubquery(TSourcePtr source, const TString& alias, bool inSubquery, int ensureTupleSize = -1);
- TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleIndex = -1);
+ TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleIndex = -1);
TNodePtr BuildSourceNode(TPosition pos, TSourcePtr source, bool checkExist = false);
- TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources);
+ TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources);
TSourcePtr BuildFakeSource(TPosition pos);
TSourcePtr BuildNodeSource(TPosition pos, const TNodePtr& node);
TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, bool stream, const TString& label = TString());
diff --git a/ydb/library/yql/sql/v0/query.cpp b/ydb/library/yql/sql/v0/query.cpp
index 7752b28b01..26a882d727 100644
--- a/ydb/library/yql/sql/v0/query.cpp
+++ b/ydb/library/yql/sql/v0/query.cpp
@@ -1,6 +1,6 @@
#include "node.h"
#include "context.h"
-
+
#include <ydb/library/yql/ast/yql_type_string.h>
#include <ydb/library/yql/core/yql_callable_names.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
@@ -8,8 +8,8 @@
#include <library/cpp/charset/ci_string.h>
-#include <util/digest/fnv.h>
-
+#include <util/digest/fnv.h>
+
using namespace NYql;
namespace NSQLTranslationV0 {
@@ -334,8 +334,8 @@ public:
return false;
}
TNodePtr option;
- if (used.insert(hint).second) {
- option = Y(BuildQuotedAtom(Pos, hint));
+ if (used.insert(hint).second) {
+ option = Y(BuildQuotedAtom(Pos, hint));
}
if (option) {
Nodes.push_back(Q(option));
@@ -982,20 +982,20 @@ public:
}
if (ctx.Settings.Mode != NSQLTranslation::ESqlMode::LIBRARY) {
- auto configSource = Y("DataSource", BuildQuotedAtom(Pos, TString(ConfigProviderName)));
- auto resultSink = Y("DataSink", BuildQuotedAtom(Pos, TString(ResultProviderName)));
+ auto configSource = Y("DataSource", BuildQuotedAtom(Pos, TString(ConfigProviderName)));
+ auto resultSink = Y("DataSink", BuildQuotedAtom(Pos, TString(ResultProviderName)));
for (const auto& warningPragma : ctx.WarningPolicy.GetRules()) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()),
BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction()))))));
- }
+ }
- if (ctx.ResultSizeLimit > 0) {
+ if (ctx.ResultSizeLimit > 0) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", resultSink,
- BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit)))));
- }
-
+ BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit)))));
+ }
+
if (!ctx.PragmaPullUpFlatMapOverJoin) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "DisablePullUpFlatMapOverJoin"))));
@@ -1024,8 +1024,8 @@ public:
Add(Y("let", ref, Y("Nth", *subqueryAliasPtr, Q("1"))));
}
} else {
- const auto& ref = block->GetLabel();
- Add(Y("let", ref ? ref : "world", block));
+ const auto& ref = block->GetLabel();
+ Add(Y("let", ref ? ref : "world", block));
}
}
@@ -1051,56 +1051,56 @@ public:
return !hasError;
}
- TPtr DoClone() const final {
- return {};
- }
-private:
+ TPtr DoClone() const final {
+ return {};
+ }
+private:
TVector<TNodePtr> Blocks;
const bool TopLevel;
-};
+};
TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel) {
return new TYqlProgramNode(pos, blocks, topLevel);
-}
-
+}
+
class TPragmaNode final: public INode {
-public:
+public:
TPragmaNode(TPosition pos, const TString& prefix, const TString& name, const TVector<TDeferredAtom>& values, bool valueDefault)
: INode(pos)
- , Prefix(prefix)
- , Name(name)
- , Values(values)
- , ValueDefault(valueDefault)
+ , Prefix(prefix)
+ , Name(name)
+ , Values(values)
+ , ValueDefault(valueDefault)
{
FakeSource = BuildFakeSource(pos);
}
-
- bool DoInit(TContext& ctx, ISource* src) override {
+
+ bool DoInit(TContext& ctx, ISource* src) override {
Y_UNUSED(src);
- TString serviceName;
- TString cluster;
+ TString serviceName;
+ TString cluster;
if (std::find(Providers.cbegin(), Providers.cend(), Prefix) != Providers.cend()) {
- cluster = "$all";
- serviceName = Prefix;
+ cluster = "$all";
+ serviceName = Prefix;
} else {
serviceName = *ctx.GetClusterProvider(Prefix, cluster);
}
- auto datasource = Y("DataSource", BuildQuotedAtom(Pos, serviceName));
- if (Prefix != ConfigProviderName) {
- datasource = L(datasource, BuildQuotedAtom(Pos, cluster));
- }
-
+ auto datasource = Y("DataSource", BuildQuotedAtom(Pos, serviceName));
+ if (Prefix != ConfigProviderName) {
+ datasource = L(datasource, BuildQuotedAtom(Pos, cluster));
+ }
+
Node = Y();
Node = L(Node, AstNode(TString(ConfigureName)));
Node = L(Node, AstNode(TString(TStringBuf("world"))));
Node = L(Node, datasource);
-
+
if (Name == TStringBuf("flags")) {
- for (ui32 i = 0; i < Values.size(); ++i) {
+ for (ui32 i = 0; i < Values.size(); ++i) {
Node = L(Node, Values[i].Build());
- }
- }
+ }
+ }
else if (Name == TStringBuf("AddFileByUrl") || Name == TStringBuf("AddFolderByUrl") || Name == TStringBuf("ImportUdfs")) {
Node = L(Node, BuildQuotedAtom(Pos, Name));
for (ui32 i = 0; i < Values.size(); ++i) {
@@ -1110,15 +1110,15 @@ public:
else if (Name == TStringBuf("auth")) {
Node = L(Node, BuildQuotedAtom(Pos, "Auth"));
Node = L(Node, Values.empty() ? BuildQuotedAtom(Pos, TString()) : Values.front().Build());
- }
- else {
+ }
+ else {
Node = L(Node, BuildQuotedAtom(Pos, "Attr"));
Node = L(Node, BuildQuotedAtom(Pos, Name));
- if (!ValueDefault) {
+ if (!ValueDefault) {
Node = L(Node, Values.empty() ? BuildQuotedAtom(Pos, TString()) : Values.front().Build());
- }
- }
-
+ }
+ }
+
ctx.PushBlockShortcuts();
if (!Node->Init(ctx, FakeSource.Get())) {
return false;
@@ -1135,18 +1135,18 @@ public:
TPtr DoClone() const final {
return {};
}
-
+
private:
- TString Prefix;
- TString Name;
+ TString Prefix;
+ TString Name;
TVector<TDeferredAtom> Values;
- bool ValueDefault;
+ bool ValueDefault;
TNodePtr Node;
TSourcePtr FakeSource;
};
TNodePtr BuildPragma(TPosition pos, const TString& prefix, const TString& name, const TVector<TDeferredAtom>& values, bool valueDefault) {
- return new TPragmaNode(pos, prefix, name, values, valueDefault);
+ return new TPragmaNode(pos, prefix, name, values, valueDefault);
}
class TSqlLambda final: public TAstListNode {
diff --git a/ydb/library/yql/sql/v0/select.cpp b/ydb/library/yql/sql/v0/select.cpp
index ef47023be1..dc7ac37d70 100644
--- a/ydb/library/yql/sql/v0/select.cpp
+++ b/ydb/library/yql/sql/v0/select.cpp
@@ -18,7 +18,7 @@ public:
, Source(std::move(source))
, Alias(alias)
, InSubquery(inSubquery)
- , EnsureTupleSize(ensureTupleSize)
+ , EnsureTupleSize(ensureTupleSize)
{
YQL_ENSURE(!Alias.empty());
}
@@ -46,9 +46,9 @@ public:
if (!source) {
return false;
}
- if (EnsureTupleSize != -1) {
- source = Y("EnsureTupleSize", source, Q(ToString(EnsureTupleSize)));
- }
+ if (EnsureTupleSize != -1) {
+ source = Y("EnsureTupleSize", source, Q(ToString(EnsureTupleSize)));
+ }
Node = Y("let", Alias, Y("block", Q(L(tables, Y("return", Q(Y("world", source)))))));
IsUsed = true;
@@ -81,7 +81,7 @@ protected:
TNodePtr Node;
const TString Alias;
const bool InSubquery;
- const int EnsureTupleSize;
+ const int EnsureTupleSize;
bool IsUsed = false;
};
@@ -191,8 +191,8 @@ public:
return false;
}
- TNodePtr BuildFilter(TContext& ctx, const TString& label, const TNodePtr& groundNode) override {
- Y_UNUSED(ctx);
+ TNodePtr BuildFilter(TContext& ctx, const TString& label, const TNodePtr& groundNode) override {
+ Y_UNUSED(ctx);
Y_UNUSED(label);
Y_UNUSED(groundNode);
return nullptr;
@@ -258,7 +258,7 @@ public:
members = L(members, BuildQuotedAtom(Pos, column));
}
- return Y(ctx.UseUnordered(*this) ? "OrderedMap" : "Map", nodeAst, BuildLambda(Pos, Y("row"), Y("SelectMembers", "row", Q(members))));
+ return Y(ctx.UseUnordered(*this) ? "OrderedMap" : "Map", nodeAst, BuildLambda(Pos, Y("row"), Y("SelectMembers", "row", Q(members))));
}
}
@@ -317,11 +317,11 @@ protected:
return Source->IsStream();
}
- bool IsOrdered() const override {
- Y_VERIFY_DEBUG(Source);
- return Source->IsOrdered();
- }
-
+ bool IsOrdered() const override {
+ Y_VERIFY_DEBUG(Source);
+ return Source->IsOrdered();
+ }
+
TWriteSettings GetWriteSettings() const override {
Y_VERIFY_DEBUG(Source);
return Source->GetWriteSettings();
@@ -388,115 +388,115 @@ protected:
TColumns Columns;
};
-class TMuxSource: public ISource {
-public:
- TMuxSource(TPosition pos, TVector<TSourcePtr>&& sources)
- : ISource(pos)
- , Sources(std::move(sources))
- {
- YQL_ENSURE(Sources.size() > 1);
- }
-
- void AllColumns() final {
- for (auto& source: Sources) {
- source->AllColumns();
- }
- }
-
- const TColumns* GetColumns() const final {
- // Columns are equal in all sources. Return from the first one
- return Sources.front()->GetColumns();
- }
-
- void GetInputTables(TTableList& tableList) const final {
- for (auto& source: Sources) {
- source->GetInputTables(tableList);
- }
- ISource::GetInputTables(tableList);
- }
-
- bool IsStream() const final {
- return AnyOf(Sources, [] (const TSourcePtr& s) { return s->IsStream(); });
- }
-
- bool DoInit(TContext& ctx, ISource* src) final {
- for (auto& source: Sources) {
- if (AsInner) {
- source->UseAsInner();
- }
-
- ctx.PushBlockShortcuts();
- if (src) {
- src->AddDependentSource(source.Get());
- }
- if (!source->Init(ctx, src)) {
- return false;
- }
- if (!source->InitFilters(ctx)) {
- return false;
- }
- FiltersGrounds.push_back(ctx.GroundBlockShortcuts(Pos));
- }
- return true;
- }
-
- TMaybe<bool> AddColumn(TContext& ctx, TColumnNode& column) final {
- for (auto& source: Sources) {
- if (!source->AddColumn(ctx, column)) {
- return {};
- }
- }
- return true;
- }
-
- TNodePtr Build(TContext& ctx) final {
- TNodePtr block;
- auto muxArgs = Y();
- for (size_t i = 0; i < Sources.size(); ++i) {
- auto& source = Sources[i];
- auto input = source->Build(ctx);
- auto ref = ctx.MakeName("src");
- muxArgs->Add(ref);
- if (block) {
- block = L(block, Y("let", ref, input));
- } else {
- block = Y(Y("let", ref, input));
- }
- auto filter = source->BuildFilter(ctx, ref, FiltersGrounds[i]);
- if (filter) {
- block = L(block, Y("let", ref, filter));
- }
- }
- return GroundWithExpr(block, Y("Mux", Q(muxArgs)));
- }
-
- bool AddFilter(TContext& ctx, TNodePtr filter) final {
- Y_UNUSED(filter);
- ctx.Error() << "Filter is not allowed for multiple sources";
- return false;
- }
-
- TPtr DoClone() const final {
- // Don't clone FiltersGrounds container because it will be initialized in DoInit of cloned object
- return new TMuxSource(Pos, CloneContainer(Sources));
- }
-
-protected:
- TVector<TSourcePtr> Sources;
- TVector<TNodePtr> FiltersGrounds;
-};
-
-TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources) {
- return new TMuxSource(pos, std::move(sources));
-}
-
+class TMuxSource: public ISource {
+public:
+ TMuxSource(TPosition pos, TVector<TSourcePtr>&& sources)
+ : ISource(pos)
+ , Sources(std::move(sources))
+ {
+ YQL_ENSURE(Sources.size() > 1);
+ }
+
+ void AllColumns() final {
+ for (auto& source: Sources) {
+ source->AllColumns();
+ }
+ }
+
+ const TColumns* GetColumns() const final {
+ // Columns are equal in all sources. Return from the first one
+ return Sources.front()->GetColumns();
+ }
+
+ void GetInputTables(TTableList& tableList) const final {
+ for (auto& source: Sources) {
+ source->GetInputTables(tableList);
+ }
+ ISource::GetInputTables(tableList);
+ }
+
+ bool IsStream() const final {
+ return AnyOf(Sources, [] (const TSourcePtr& s) { return s->IsStream(); });
+ }
+
+ bool DoInit(TContext& ctx, ISource* src) final {
+ for (auto& source: Sources) {
+ if (AsInner) {
+ source->UseAsInner();
+ }
+
+ ctx.PushBlockShortcuts();
+ if (src) {
+ src->AddDependentSource(source.Get());
+ }
+ if (!source->Init(ctx, src)) {
+ return false;
+ }
+ if (!source->InitFilters(ctx)) {
+ return false;
+ }
+ FiltersGrounds.push_back(ctx.GroundBlockShortcuts(Pos));
+ }
+ return true;
+ }
+
+ TMaybe<bool> AddColumn(TContext& ctx, TColumnNode& column) final {
+ for (auto& source: Sources) {
+ if (!source->AddColumn(ctx, column)) {
+ return {};
+ }
+ }
+ return true;
+ }
+
+ TNodePtr Build(TContext& ctx) final {
+ TNodePtr block;
+ auto muxArgs = Y();
+ for (size_t i = 0; i < Sources.size(); ++i) {
+ auto& source = Sources[i];
+ auto input = source->Build(ctx);
+ auto ref = ctx.MakeName("src");
+ muxArgs->Add(ref);
+ if (block) {
+ block = L(block, Y("let", ref, input));
+ } else {
+ block = Y(Y("let", ref, input));
+ }
+ auto filter = source->BuildFilter(ctx, ref, FiltersGrounds[i]);
+ if (filter) {
+ block = L(block, Y("let", ref, filter));
+ }
+ }
+ return GroundWithExpr(block, Y("Mux", Q(muxArgs)));
+ }
+
+ bool AddFilter(TContext& ctx, TNodePtr filter) final {
+ Y_UNUSED(filter);
+ ctx.Error() << "Filter is not allowed for multiple sources";
+ return false;
+ }
+
+ TPtr DoClone() const final {
+ // Don't clone FiltersGrounds container because it will be initialized in DoInit of cloned object
+ return new TMuxSource(Pos, CloneContainer(Sources));
+ }
+
+protected:
+ TVector<TSourcePtr> Sources;
+ TVector<TNodePtr> FiltersGrounds;
+};
+
+TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources) {
+ return new TMuxSource(pos, std::move(sources));
+}
+
class TSubqueryRefNode: public IRealSource {
public:
- TSubqueryRefNode(const TNodePtr& subquery, const TString& alias, int tupleIndex)
+ TSubqueryRefNode(const TNodePtr& subquery, const TString& alias, int tupleIndex)
: IRealSource(subquery->GetPos())
, Subquery(subquery)
, Alias(alias)
- , TupleIndex(tupleIndex)
+ , TupleIndex(tupleIndex)
{
YQL_ENSURE(subquery->GetSource());
}
@@ -513,9 +513,9 @@ public:
}
Columns = *Subquery->GetSource()->GetColumns();
Node = BuildAtom(Pos, Alias, TNodeFlags::Default);
- if (TupleIndex != -1) {
- Node = Y("Nth", Node, Q(ToString(TupleIndex)));
- }
+ if (TupleIndex != -1) {
+ Node = Y("Nth", Node, Q(ToString(TupleIndex)));
+ }
if (!Node->Init(ctx, src)) {
return false;
}
@@ -549,18 +549,18 @@ public:
}
TPtr DoClone() const final {
- return new TSubqueryRefNode(Subquery, Alias, TupleIndex);
+ return new TSubqueryRefNode(Subquery, Alias, TupleIndex);
}
protected:
TNodePtr Subquery;
const TString Alias;
- const int TupleIndex;
+ const int TupleIndex;
TNodePtr Node;
};
-TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleIndex) {
- return new TSubqueryRefNode(std::move(subquery), alias, tupleIndex);
+TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleIndex) {
+ return new TSubqueryRefNode(std::move(subquery), alias, tupleIndex);
}
class TTableSource: public IRealSource {
@@ -620,9 +620,9 @@ public:
auto sampleSettings = Q(Y(Q(modeName), Y("EvaluateAtom", Y("ToString", samplingRate)), Y("EvaluateAtom", Y("ToString", samplingSeed))));
auto sampleOption = Q(Y(Q("sample"), sampleSettings));
if (Table.Options) {
- if (!Table.Options->Init(ctx, this)) {
- return false;
- }
+ if (!Table.Options->Init(ctx, this)) {
+ return false;
+ }
Table.Options = L(Table.Options, sampleOption);
} else {
Table.Options = Y(sampleOption);
@@ -821,17 +821,17 @@ public:
return nullptr;
}
- auto keysTuple = Y();
- if (Keys.size() == 1) {
- keysTuple = Y("Member", "row", BuildQuotedAtom(Pos, *Keys.back()->GetColumnName()));
- }
- else {
+ auto keysTuple = Y();
+ if (Keys.size() == 1) {
+ keysTuple = Y("Member", "row", BuildQuotedAtom(Pos, *Keys.back()->GetColumnName()));
+ }
+ else {
for (const auto& key: Keys) {
keysTuple = L(keysTuple, Y("Member", "row", BuildQuotedAtom(Pos, *key->GetColumnName())));
}
keysTuple = Q(keysTuple);
}
- auto extractKey = Y("SqlExtractKey", "row", BuildLambda(Pos, Y("row"), keysTuple));
+ auto extractKey = Y("SqlExtractKey", "row", BuildLambda(Pos, Y("row"), keysTuple));
auto extractKeyLambda = BuildLambda(Pos, Y("row"), extractKey);
TNodePtr processPartitions;
@@ -850,7 +850,7 @@ public:
}
case ReduceMode::ByPartition: {
processPartitions = Y("SqlReduce", "partitionStream", extractKeyLambda, Udf,
- BuildLambda(Pos, Y("row"), GroundWithExpr(ExprGround, Args[0])));
+ BuildLambda(Pos, Y("row"), GroundWithExpr(ExprGround, Args[0])));
break;
}
default:
@@ -860,19 +860,19 @@ public:
TNodePtr sortDirection;
auto sortKeySelector = OrderByGround;
FillSortParts(OrderBy, sortDirection, sortKeySelector);
- if (!OrderBy.empty()) {
- sortKeySelector = BuildLambda(Pos, Y("row"), Y("SqlExtractKey", "row", sortKeySelector));
- }
+ if (!OrderBy.empty()) {
+ sortKeySelector = BuildLambda(Pos, Y("row"), Y("SqlExtractKey", "row", sortKeySelector));
+ }
auto partitionByKey = Y(Mode == ReduceMode::ByAll ? "PartitionByKey" : "PartitionsByKeys", "core", extractKeyLambda,
sortDirection, sortKeySelector, BuildLambda(Pos, Y("partitionStream"), processPartitions));
auto block(Y(Y("let", "core", input)));
- auto filter = Source->BuildFilter(ctx, "core", FiltersGround);
+ auto filter = Source->BuildFilter(ctx, "core", FiltersGround);
if (filter) {
block = L(block, Y("let", "core", filter));
}
- block = L(block, Y("let", "core", Y("AutoDemuxList", partitionByKey)));
+ block = L(block, Y("let", "core", Y("AutoDemuxList", partitionByKey)));
if (Having) {
block = L(block, Y("let", "core",
Y("Filter", "core", BuildLambda(Pos, Y("row"), GroundWithExpr(HavingGround, Y("Coalesce", Having, Y("Bool", Q("false"))))))
@@ -980,7 +980,7 @@ public:
TNodePtr Build(TContext& ctx) override {
auto input = Source->Build(ctx);
auto block(Y(Y("let", "composite", input)));
- auto filter = Source->BuildFilter(ctx, "composite", FiltersGround);
+ auto filter = Source->BuildFilter(ctx, "composite", FiltersGround);
if (filter) {
block = L(block, Y("let", "composite", filter));
}
@@ -1009,10 +1009,10 @@ public:
return Subselects.front()->BuildSort(ctx, label);
}
- bool IsOrdered() const override {
- return Subselects.front()->IsOrdered();
- }
-
+ bool IsOrdered() const override {
+ return Subselects.front()->IsOrdered();
+ }
+
const TColumns* GetColumns() const override{
return Subselects.front()->GetColumns();
}
@@ -1164,8 +1164,8 @@ public:
HavingGround = ctx.GroundBlockShortcuts(Pos);
src->AddWindowSpecs(WinSpecs);
- if (!InitSelect(ctx, src, isJoin, hasError)) {
- return false;
+ if (!InitSelect(ctx, src, isJoin, hasError)) {
+ return false;
}
src->FinishColumns();
@@ -1236,10 +1236,10 @@ public:
TNodePtr terms = BuildColumnsTerms(ctx);
- bool ordered = ctx.UseUnordered(*this);
+ bool ordered = ctx.UseUnordered(*this);
auto block(Y(Y("let", "core", input)));
if (Flatten) {
- block = L(block, Y("let", "core", Y(ordered ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), Flatten, "res"))));
+ block = L(block, Y("let", "core", Y(ordered ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), Flatten, "res"))));
}
if (PreaggregatedMap) {
block = L(block, Y("let", "core", Y("FlatMap", "core", BuildLambda(Pos, Y("row"), PreaggregatedMap))));
@@ -1249,13 +1249,13 @@ public:
} else if (Source->IsCompositeSource() && !Columns.QualifiedAll) {
block = L(block, Y("let", "origcore", "core"));
}
- auto filter = Source->BuildFilter(ctx, "core", FiltersGround);
+ auto filter = Source->BuildFilter(ctx, "core", FiltersGround);
if (filter) {
block = L(block, Y("let", "core", filter));
}
if (Aggregate) {
block = L(block, Y("let", "core", Aggregate));
- ordered = false;
+ ordered = false;
}
if (PrewindowMap) {
block = L(block, Y("let", "core", PrewindowMap));
@@ -1263,7 +1263,7 @@ public:
if (CalcOverWindow) {
block = L(block, Y("let", "core", CalcOverWindow));
}
- block = L(block, Y("let", "core", Y("EnsurePersistable", Y(ordered ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), terms, "res")))));
+ block = L(block, Y("let", "core", Y("EnsurePersistable", Y(ordered ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), terms, "res")))));
return Y("block", Q(L(block, Y("return", "core"))));
}
@@ -1277,17 +1277,17 @@ public:
}
bool IsSelect() const override {
- return true;
+ return true;
}
bool IsStream() const override {
return Stream;
}
- bool IsOrdered() const override {
- return !OrderBy.empty();
- }
-
+ bool IsOrdered() const override {
+ return !OrderBy.empty();
+ }
+
TWriteSettings GetWriteSettings() const override {
return Settings;
}
@@ -1330,7 +1330,7 @@ public:
for (auto cur: WinSpecs) {
newSpecs.emplace(cur.first, cur.second->Clone());
}
- return new TSelectCore(Pos, Source->CloneSource(), CloneContainer(GroupByExpr),
+ return new TSelectCore(Pos, Source->CloneSource(), CloneContainer(GroupByExpr),
CloneContainer(GroupBy), CloneContainer(OrderBy), SafeClone(Having), newSpecs, SafeClone(HoppingWindowSpec),
CloneContainer(Terms), Distinct, Without, Stream, Settings);
}
@@ -1516,8 +1516,8 @@ private:
TNodePtr BuildColumnsTerms(TContext& ctx) {
TNodePtr terms;
if (Columns.All) {
- Y_VERIFY_DEBUG(Columns.List.empty());
- terms = PrepareWithout(Y());
+ Y_VERIFY_DEBUG(Columns.List.empty());
+ terms = PrepareWithout(Y());
if (ctx.EnableSystemColumns) {
terms = L(terms, Y("let", "res", Y("AsList", Y("RemoveSystemMembers", "row"))));
} else {
@@ -1654,41 +1654,41 @@ private:
const TWriteSettings Settings;
};
-class TProcessSource: public IRealSource {
-public:
- TProcessSource(
- TPosition pos,
- TSourcePtr source,
- TNodePtr with,
+class TProcessSource: public IRealSource {
+public:
+ TProcessSource(
+ TPosition pos,
+ TSourcePtr source,
+ TNodePtr with,
TVector<TNodePtr>&& terms,
- bool listCall,
+ bool listCall,
bool stream,
const TWriteSettings& settings
- )
- : IRealSource(pos)
- , Source(std::move(source))
- , With(with)
+ )
+ : IRealSource(pos)
+ , Source(std::move(source))
+ , With(with)
, Terms(std::move(terms))
- , ListCall(listCall)
+ , ListCall(listCall)
, Stream(stream)
, Settings(settings)
- {
- }
-
+ {
+ }
+
void GetInputTables(TTableList& tableList) const override {
Source->GetInputTables(tableList);
ISource::GetInputTables(tableList);
- }
-
- bool DoInit(TContext& ctx, ISource* initSrc) override {
- if (AsInner) {
- Source->UseAsInner();
- }
-
- if (!Source->Init(ctx, initSrc)) {
- return false;
- }
-
+ }
+
+ bool DoInit(TContext& ctx, ISource* initSrc) override {
+ if (AsInner) {
+ Source->UseAsInner();
+ }
+
+ if (!Source->Init(ctx, initSrc)) {
+ return false;
+ }
+
if (Stream && !Source->IsStream()) {
ctx.Error(Pos) << "PROCESS STREAM is unsupported for non-streaming sources";
return false;
@@ -1699,7 +1699,7 @@ public:
return false;
}
- auto src = Source.Get();
+ auto src = Source.Get();
if (!With) {
src->AllColumns();
Columns.SetAll();
@@ -1707,172 +1707,172 @@ public:
return true;
}
- /// grouped expressions are available in filters
- ctx.PushBlockShortcuts();
- if (!Source->InitFilters(ctx)) {
- return false;
- }
- FiltersGround = ctx.GroundBlockShortcuts(Pos);
-
- // Use fake source in case of list process to restrict column access.
- TSourcePtr fakeSource;
- if (ListCall) {
- fakeSource = BuildFakeSource(src->GetPos());
- src->AllColumns();
- }
-
- auto processSource = ListCall ? fakeSource.Get() : src;
- Y_VERIFY_DEBUG(processSource != nullptr);
-
- ctx.PushBlockShortcuts();
- if (!With->Init(ctx, processSource)) {
- return false;
- }
- if (With->GetLabel().empty()) {
- Columns.SetAll();
- } else {
- if (ListCall) {
- ctx.Error(With->GetPos()) << "Label is not allowed to use with $ROWS";
- return false;
- }
-
+ /// grouped expressions are available in filters
+ ctx.PushBlockShortcuts();
+ if (!Source->InitFilters(ctx)) {
+ return false;
+ }
+ FiltersGround = ctx.GroundBlockShortcuts(Pos);
+
+ // Use fake source in case of list process to restrict column access.
+ TSourcePtr fakeSource;
+ if (ListCall) {
+ fakeSource = BuildFakeSource(src->GetPos());
+ src->AllColumns();
+ }
+
+ auto processSource = ListCall ? fakeSource.Get() : src;
+ Y_VERIFY_DEBUG(processSource != nullptr);
+
+ ctx.PushBlockShortcuts();
+ if (!With->Init(ctx, processSource)) {
+ return false;
+ }
+ if (With->GetLabel().empty()) {
+ Columns.SetAll();
+ } else {
+ if (ListCall) {
+ ctx.Error(With->GetPos()) << "Label is not allowed to use with $ROWS";
+ return false;
+ }
+
Columns.Add(&With->GetLabel(), false);
- }
-
- bool hasError = false;
- auto produce = Y(ListCall ? "SqlProcess" : "Apply", With);
- TMaybe<ui32> listPosIndex;
- ui32 termIndex = 0;
- for (auto& term: Terms) {
- if (ListCall) {
- if (auto atom = dynamic_cast<TAstAtomNode*>(term.Get())) {
- if (atom->GetContent() == "inputRowsList") {
- listPosIndex = termIndex;
- }
- }
- }
- ++termIndex;
-
- if (!term->GetLabel().empty()) {
- ctx.Error(term->GetPos()) << "Labels are not allowed for PROCESS terms";
- hasError = true;
- continue;
- }
-
- if (!term->Init(ctx, processSource)) {
- hasError = true;
- continue;
- }
-
- produce = L(produce, term);
- }
-
- if (ListCall) {
- produce = L(produce, Q(ToString(*listPosIndex)));
- }
-
- if (!produce->Init(ctx, src)) {
- hasError = true;
- }
- produce = ctx.GroundBlockShortcutsForExpr(produce);
-
- TVector<TNodePtr>(1, produce).swap(Terms);
-
- src->FinishColumns();
-
- if (hasError) {
- return false;
- }
-
- return true;
- }
-
- TNodePtr Build(TContext& ctx) override {
- auto input = Source->Build(ctx);
- if (!input) {
- return nullptr;
- }
-
+ }
+
+ bool hasError = false;
+ auto produce = Y(ListCall ? "SqlProcess" : "Apply", With);
+ TMaybe<ui32> listPosIndex;
+ ui32 termIndex = 0;
+ for (auto& term: Terms) {
+ if (ListCall) {
+ if (auto atom = dynamic_cast<TAstAtomNode*>(term.Get())) {
+ if (atom->GetContent() == "inputRowsList") {
+ listPosIndex = termIndex;
+ }
+ }
+ }
+ ++termIndex;
+
+ if (!term->GetLabel().empty()) {
+ ctx.Error(term->GetPos()) << "Labels are not allowed for PROCESS terms";
+ hasError = true;
+ continue;
+ }
+
+ if (!term->Init(ctx, processSource)) {
+ hasError = true;
+ continue;
+ }
+
+ produce = L(produce, term);
+ }
+
+ if (ListCall) {
+ produce = L(produce, Q(ToString(*listPosIndex)));
+ }
+
+ if (!produce->Init(ctx, src)) {
+ hasError = true;
+ }
+ produce = ctx.GroundBlockShortcutsForExpr(produce);
+
+ TVector<TNodePtr>(1, produce).swap(Terms);
+
+ src->FinishColumns();
+
+ if (hasError) {
+ return false;
+ }
+
+ return true;
+ }
+
+ TNodePtr Build(TContext& ctx) override {
+ auto input = Source->Build(ctx);
+ if (!input) {
+ return nullptr;
+ }
+
if (!With) {
- return input;
- }
-
- TString inputLabel = ListCall ? "inputRowsList" : "core";
-
- auto block(Y(Y("let", inputLabel, input)));
-
- auto filter = Source->BuildFilter(ctx, inputLabel, FiltersGround);
- if (filter) {
- block = L(block, Y("let", inputLabel, filter));
- }
-
- if (ListCall) {
- block = L(block, Y("let", "core", Terms[0]));
- } else {
- auto terms = BuildColumnsTerms(ctx);
- block = L(block, Y("let", "core", Y(ctx.UseUnordered(*this) ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), terms, "res"))));
- }
- block = L(block, Y("let", "core", Y("AutoDemuxList", Y("EnsurePersistable", "core"))));
- return Y("block", Q(L(block, Y("return", "core"))));
- }
-
- bool IsSelect() const override {
- return false;
- }
-
+ return input;
+ }
+
+ TString inputLabel = ListCall ? "inputRowsList" : "core";
+
+ auto block(Y(Y("let", inputLabel, input)));
+
+ auto filter = Source->BuildFilter(ctx, inputLabel, FiltersGround);
+ if (filter) {
+ block = L(block, Y("let", inputLabel, filter));
+ }
+
+ if (ListCall) {
+ block = L(block, Y("let", "core", Terms[0]));
+ } else {
+ auto terms = BuildColumnsTerms(ctx);
+ block = L(block, Y("let", "core", Y(ctx.UseUnordered(*this) ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), terms, "res"))));
+ }
+ block = L(block, Y("let", "core", Y("AutoDemuxList", Y("EnsurePersistable", "core"))));
+ return Y("block", Q(L(block, Y("return", "core"))));
+ }
+
+ bool IsSelect() const override {
+ return false;
+ }
+
bool IsStream() const override {
return Stream;
}
TWriteSettings GetWriteSettings() const override {
return Settings;
- }
-
- TNodePtr DoClone() const final {
- return new TProcessSource(Pos, Source->CloneSource(), SafeClone(With),
+ }
+
+ TNodePtr DoClone() const final {
+ return new TProcessSource(Pos, Source->CloneSource(), SafeClone(With),
CloneContainer(Terms), ListCall, Stream, Settings);
- }
-
-private:
- TNodePtr BuildColumnsTerms(TContext& ctx) {
+ }
+
+private:
+ TNodePtr BuildColumnsTerms(TContext& ctx) {
Y_UNUSED(ctx);
- TNodePtr terms;
- Y_VERIFY_DEBUG(Terms.size() == 1);
- if (Columns.All) {
- terms = Y(Y("let", "res", Y("ToSequence", Terms.front())));
- } else {
- Y_VERIFY_DEBUG(Columns.List.size() == Terms.size());
- terms = TermsGround ? TermsGround : Y();
- terms = L(terms, Y("let", "res",
- L(Y("AsStruct"), Q(Y(BuildQuotedAtom(Pos, Columns.List.front()), Terms.front())))));
- terms = L(terms, Y("let", "res", Y("Just", "res")));
- }
- return terms;
- }
-
-private:
- TSourcePtr Source;
- TNodePtr With;
- TNodePtr FiltersGround;
- TNodePtr TermsGround;
- TVector<TNodePtr> Terms;
- const bool ListCall;
+ TNodePtr terms;
+ Y_VERIFY_DEBUG(Terms.size() == 1);
+ if (Columns.All) {
+ terms = Y(Y("let", "res", Y("ToSequence", Terms.front())));
+ } else {
+ Y_VERIFY_DEBUG(Columns.List.size() == Terms.size());
+ terms = TermsGround ? TermsGround : Y();
+ terms = L(terms, Y("let", "res",
+ L(Y("AsStruct"), Q(Y(BuildQuotedAtom(Pos, Columns.List.front()), Terms.front())))));
+ terms = L(terms, Y("let", "res", Y("Just", "res")));
+ }
+ return terms;
+ }
+
+private:
+ TSourcePtr Source;
+ TNodePtr With;
+ TNodePtr FiltersGround;
+ TNodePtr TermsGround;
+ TVector<TNodePtr> Terms;
+ const bool ListCall;
const bool Stream;
const TWriteSettings Settings;
-};
-
-TSourcePtr BuildProcess(
- TPosition pos,
- TSourcePtr source,
- TNodePtr with,
- TVector<TNodePtr>&& terms,
- bool listCall,
+};
+
+TSourcePtr BuildProcess(
+ TPosition pos,
+ TSourcePtr source,
+ TNodePtr with,
+ TVector<TNodePtr>&& terms,
+ bool listCall,
bool stream,
const TWriteSettings& settings
-) {
+) {
return new TProcessSource(pos, std::move(source), with, std::move(terms), listCall, stream, settings);
-}
-
+}
+
class TNestedProxySource: public IProxySource {
public:
TNestedProxySource(TPosition pos, const TVector<TNodePtr>& groupBy, TSourcePtr source)
@@ -1900,8 +1900,8 @@ public:
return CompositeSelect ? true : Source->InitFilters(ctx);
}
- TNodePtr BuildFilter(TContext& ctx, const TString& label, const TNodePtr& groundNode) override {
- return CompositeSelect ? nullptr : Source->BuildFilter(ctx, label, groundNode);
+ TNodePtr BuildFilter(TContext& ctx, const TString& label, const TNodePtr& groundNode) override {
+ return CompositeSelect ? nullptr : Source->BuildFilter(ctx, label, groundNode);
}
bool IsCompositeSource() const override {
@@ -1974,7 +1974,7 @@ TSourcePtr BuildSelectCore(
/// actualy no big idea to use grouping function in this case (result allways 0)
auto contentPtr = groupBy.front()->ContentListPtr();
TSourcePtr proxySource = new TNestedProxySource(pos, *contentPtr, std::move(source));
- return BuildSelectCore(ctx, pos, std::move(proxySource), groupByExpr, *contentPtr, orderBy, having, std::move(winSpecs),
+ return BuildSelectCore(ctx, pos, std::move(proxySource), groupByExpr, *contentPtr, orderBy, having, std::move(winSpecs),
hoppingWindowSpec, std::move(terms), distinct, std::move(without), stream, settings);
}
/// \todo some smart merge logic, generalize common part of grouping (expr, flatten, etc)?
@@ -2005,7 +2005,7 @@ TSourcePtr BuildSelectCore(
std::swap(terms, termsCopy);
}
totalGroups += contentPtr->size();
- TSelectCore* selectCore = new TSelectCore(pos, std::move(proxySource), CloneContainer(groupByExpr),
+ TSelectCore* selectCore = new TSelectCore(pos, std::move(proxySource), CloneContainer(groupByExpr),
*contentPtr, orderBy, SafeClone(having), winSpecs, hoppingWindowSpec, terms, distinct, without, stream, settings);
subselects.emplace_back(selectCore);
}
@@ -2336,9 +2336,9 @@ public:
}
node = L(node, Y("let", "output", output));
if (WriteResult) {
- if (!Source->IsOrdered() && ctx.UseUnordered(*Source)) {
- node = L(node, Y("let", "output", Y("Unordered", "output")));
- }
+ if (!Source->IsOrdered() && ctx.UseUnordered(*Source)) {
+ node = L(node, Y("let", "output", Y("Unordered", "output")));
+ }
auto writeResult(BuildWriteResult(Pos, "output", settings, clusters));
if (!writeResult->Init(ctx, src)) {
return false;
diff --git a/ydb/library/yql/sql/v0/sql.cpp b/ydb/library/yql/sql/v0/sql.cpp
index b028fc0439..f006f3b7f4 100644
--- a/ydb/library/yql/sql/v0/sql.cpp
+++ b/ydb/library/yql/sql/v0/sql.cpp
@@ -447,18 +447,18 @@ TVector<TString> GetContextHints(TContext& ctx) {
return hints;
}
-static TVector<TString> GetTableFuncHints(TStringBuf funcName) {
- TCiString func(funcName);
- if (func.StartsWith("range") || func.StartsWith("like") || func.StartsWith("regexp") || func.StartsWith("filter")
- || func.StartsWith("each")) {
-
- return TVector<TString>{"ignore_non_existing"};
- }
-
- return {};
-}
-
-
+static TVector<TString> GetTableFuncHints(TStringBuf funcName) {
+ TCiString func(funcName);
+ if (func.StartsWith("range") || func.StartsWith("like") || func.StartsWith("regexp") || func.StartsWith("filter")
+ || func.StartsWith("each")) {
+
+ return TVector<TString>{"ignore_non_existing"};
+ }
+
+ return {};
+}
+
+
static TTableRef SimpleTableRefImpl(const TRule_simple_table_ref& node, NSQLTranslation::ESqlMode mode, TTranslation& ctx) {
TMaybe<TTableRef> tr;
TString cluster;
@@ -559,7 +559,7 @@ protected:
bool ModulePath(const TRule_module_path& node, TVector<TString>& path);
bool NamedBindList(const TRule_named_bind_parameter_list& node, TVector<TNodePtr>& bindNames);
TNodePtr NamedBindParam(const TRule_named_bind_parameter& node);
- TNodePtr NamedNode(const TRule_named_nodes_stmt& rule, TVector<TString>& names);
+ TNodePtr NamedNode(const TRule_named_nodes_stmt& rule, TVector<TString>& names);
bool ImportStatement(const TRule_import_stmt& stmt, TVector<TString>* namesPtr = nullptr);
TNodePtr DoStatement(const TRule_do_stmt& stmt, bool makeLambda, const TVector<TString>& args = {});
@@ -915,7 +915,7 @@ TTableRef TSqlTranslation::TableRefImpl(const TRule_table_ref& node) {
TTableRef tr(Context().MakeName("table"), cluster, nullptr);
TPosition pos(Context().Pos());
- TVector<TString> tableHints;
+ TVector<TString> tableHints;
auto& block = node.GetBlock2();
switch (block.Alt_case()) {
case TRule_table_ref::TBlock2::kAlt1: {
@@ -939,7 +939,7 @@ TTableRef TSqlTranslation::TableRefImpl(const TRule_table_ref& node) {
args.push_back(*arg);
}
- tableHints = GetTableFuncHints(func);
+ tableHints = GetTableFuncHints(func);
tr.Keys = BuildTableKeys(pos, cluster, func, args);
break;
}
@@ -951,8 +951,8 @@ TTableRef TSqlTranslation::TableRefImpl(const TRule_table_ref& node) {
hints = TableHintsImpl(node.GetBlock3().GetRule_table_hints1(), *this);
}
- hints.insert(hints.end(), tableHints.begin(), tableHints.end());
-
+ hints.insert(hints.end(), tableHints.begin(), tableHints.end());
+
if (!hints.empty()) {
tr.Options = BuildInputOptions(pos, hints);
}
@@ -1720,8 +1720,8 @@ bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& n
const auto& rule = block.GetRule_lambda_stmt1();
switch (rule.Alt_case()) {
case TRule_lambda_stmt::kAltLambdaStmt1: {
- TVector<TString> names;
- auto nodeExpr = NamedNode(rule.GetAlt_lambda_stmt1().GetRule_named_nodes_stmt1(), names);
+ TVector<TString> names;
+ auto nodeExpr = NamedNode(rule.GetAlt_lambda_stmt1().GetRule_named_nodes_stmt1(), names);
if (!nodeExpr) {
hasError = true;
continue;
@@ -1730,24 +1730,24 @@ bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& n
hasError = true;
continue;
}
-
- if (names.size() > 1) {
- auto ref = ctx.MakeName("tie");
- exprSeq.push_back(nodeExpr->Y("EnsureTupleSize", nodeExpr, nodeExpr->Q(ToString(names.size()))));
- exprSeq.back()->SetLabel(ref);
- for (size_t i = 0; i < names.size(); ++i) {
- TNodePtr nthExpr = nodeExpr->Y("Nth", ref, nodeExpr->Q(ToString(i)));
- nthExpr->SetLabel(names[i]);
- localNames.push_back(names[i]);
- PushNamedNode(names[i], BuildAtom(nodeExpr->GetPos(), names[i], NYql::TNodeFlags::Default));
- exprSeq.push_back(nthExpr);
- }
- } else {
- nodeExpr->SetLabel(names.front());
- localNames.push_back(names.front());
- PushNamedNode(names.front(), BuildAtom(nodeExpr->GetPos(), names.front(), NYql::TNodeFlags::Default));
- exprSeq.push_back(nodeExpr);
- }
+
+ if (names.size() > 1) {
+ auto ref = ctx.MakeName("tie");
+ exprSeq.push_back(nodeExpr->Y("EnsureTupleSize", nodeExpr, nodeExpr->Q(ToString(names.size()))));
+ exprSeq.back()->SetLabel(ref);
+ for (size_t i = 0; i < names.size(); ++i) {
+ TNodePtr nthExpr = nodeExpr->Y("Nth", ref, nodeExpr->Q(ToString(i)));
+ nthExpr->SetLabel(names[i]);
+ localNames.push_back(names[i]);
+ PushNamedNode(names[i], BuildAtom(nodeExpr->GetPos(), names[i], NYql::TNodeFlags::Default));
+ exprSeq.push_back(nthExpr);
+ }
+ } else {
+ nodeExpr->SetLabel(names.front());
+ localNames.push_back(names.front());
+ PushNamedNode(names.front(), BuildAtom(nodeExpr->GetPos(), names.front(), NYql::TNodeFlags::Default));
+ exprSeq.push_back(nodeExpr);
+ }
break;
}
case TRule_lambda_stmt::kAltLambdaStmt2: {
@@ -1757,7 +1757,7 @@ bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& n
break;
}
default:
- Y_FAIL("SampleClause: does not correspond to grammar changes");
+ Y_FAIL("SampleClause: does not correspond to grammar changes");
}
}
@@ -2235,10 +2235,10 @@ TNodePtr TSqlExpression::SmartParenthesis(const TRule_smart_parenthesis& node) {
return hasUnnamed || expectTuple ? BuildTuple(pos, exprs) : BuildStructure(pos, exprs);
}
-TNodePtr TSqlTranslation::NamedNode(const TRule_named_nodes_stmt& rule, TVector<TString>& names) {
- // named_nodes_stmt: bind_parameter_list EQUALS (expr | LPAREN select_stmt RPAREN);
- BindList(rule.GetRule_bind_parameter_list1(), names);
-
+TNodePtr TSqlTranslation::NamedNode(const TRule_named_nodes_stmt& rule, TVector<TString>& names) {
+ // named_nodes_stmt: bind_parameter_list EQUALS (expr | LPAREN select_stmt RPAREN);
+ BindList(rule.GetRule_bind_parameter_list1(), names);
+
TNodePtr nodeExpr = nullptr;
switch (rule.GetBlock3().Alt_case()) {
case TRule_named_nodes_stmt::TBlock3::kAlt1: {
@@ -2880,7 +2880,7 @@ bool TSqlSelect::SortSpecificationList(const TRule_sort_specification_list& node
}
TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteSettings& settings, TPosition& selectPos) {
- // PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING call_expr (AS id_or_string)?
+ // PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING call_expr (AS id_or_string)?
// (WHERE expr)? (HAVING expr)?)?
Token(node.GetToken1());
@@ -2895,27 +2895,27 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS
if (!source) {
return nullptr;
}
- if (node.GetBlock4().size()) {
- TVector<TSourcePtr> sources(1, source);
- for (auto& s: node.GetBlock4()) {
- sources.push_back(NamedSingleSource(s.GetRule_named_single_source2()));
- if (!sources.back()) {
- return nullptr;
- }
- }
- auto pos = source->GetPos();
- source = BuildMuxSource(pos, std::move(sources));
- }
-
- bool hasUsing = node.HasBlock5();
+ if (node.GetBlock4().size()) {
+ TVector<TSourcePtr> sources(1, source);
+ for (auto& s: node.GetBlock4()) {
+ sources.push_back(NamedSingleSource(s.GetRule_named_single_source2()));
+ if (!sources.back()) {
+ return nullptr;
+ }
+ }
+ auto pos = source->GetPos();
+ source = BuildMuxSource(pos, std::move(sources));
+ }
+
+ bool hasUsing = node.HasBlock5();
if (!hasUsing) {
return BuildProcess(startPos, std::move(source), nullptr, {}, true, stream, settings);
}
- const auto& block5 = node.GetBlock5();
- if (block5.HasBlock4()) {
+ const auto& block5 = node.GetBlock5();
+ if (block5.HasBlock4()) {
TSqlExpression expr(Ctx, Mode);
- TNodePtr where = expr.Build(block5.GetBlock4().GetRule_expr2());
+ TNodePtr where = expr.Build(block5.GetBlock4().GetRule_expr2());
if (!where || !source->AddFilter(Ctx, where)) {
return nullptr;
}
@@ -2924,7 +2924,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS
Ctx.IncrementMonCounter("sql_features", stream ? "ProcessStream" : "Process");
}
- if (block5.HasBlock5()) {
+ if (block5.HasBlock5()) {
Ctx.Error() << "PROCESS does not allow HAVING yet! You may request it on yql@ maillist.";
return nullptr;
}
@@ -2935,7 +2935,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS
bool listCall = false;
TSqlCallExpr call(Ctx, Mode);
- bool initRet = call.Init(block5.GetRule_call_expr2());
+ bool initRet = call.Init(block5.GetRule_call_expr2());
if (initRet) {
call.IncCounters();
}
@@ -2975,17 +2975,17 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS
}
args = finalCall.GetArgs();
- if (block5.HasBlock3()) {
- with->SetLabel(IdOrString(block5.GetBlock3().GetRule_id_or_string2(), *this));
+ if (block5.HasBlock3()) {
+ with->SetLabel(IdOrString(block5.GetBlock3().GetRule_id_or_string2(), *this));
}
return BuildProcess(startPos, std::move(source), with, std::move(args), listCall, stream, settings);
}
TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSettings& settings, TPosition& selectPos) {
- // REDUCE named_single_source (COMMA named_single_source)* (PRESORT sort_specification_list)?
- // ON column_list USING ALL? call_expr (AS id_or_string)?
- // (WHERE expr)? (HAVING expr)?
+ // REDUCE named_single_source (COMMA named_single_source)* (PRESORT sort_specification_list)?
+ // ON column_list USING ALL? call_expr (AS id_or_string)?
+ // (WHERE expr)? (HAVING expr)?
Token(node.GetToken1());
TPosition startPos(Ctx.Pos());
if (!selectPos) {
@@ -2996,33 +2996,33 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet
if (!source) {
return {};
}
- if (node.GetBlock3().size()) {
- TVector<TSourcePtr> sources(1, source);
- for (auto& s: node.GetBlock3()) {
- sources.push_back(NamedSingleSource(s.GetRule_named_single_source2()));
- if (!sources.back()) {
- return nullptr;
- }
- }
- auto pos = source->GetPos();
- source = BuildMuxSource(pos, std::move(sources));
- }
+ if (node.GetBlock3().size()) {
+ TVector<TSourcePtr> sources(1, source);
+ for (auto& s: node.GetBlock3()) {
+ sources.push_back(NamedSingleSource(s.GetRule_named_single_source2()));
+ if (!sources.back()) {
+ return nullptr;
+ }
+ }
+ auto pos = source->GetPos();
+ source = BuildMuxSource(pos, std::move(sources));
+ }
TVector<TSortSpecificationPtr> orderBy;
- if (node.HasBlock4()) {
- if (!SortSpecificationList(node.GetBlock4().GetRule_sort_specification_list2(), orderBy)) {
+ if (node.HasBlock4()) {
+ if (!SortSpecificationList(node.GetBlock4().GetRule_sort_specification_list2(), orderBy)) {
return {};
}
}
TVector<TNodePtr> keys;
- if (!ColumnList(keys, node.GetRule_column_list6())) {
+ if (!ColumnList(keys, node.GetRule_column_list6())) {
return nullptr;
}
- if (node.HasBlock11()) {
+ if (node.HasBlock11()) {
TSqlExpression expr(Ctx, Mode);
- TNodePtr where = expr.Build(node.GetBlock11().GetRule_expr2());
+ TNodePtr where = expr.Build(node.GetBlock11().GetRule_expr2());
if (!where || !source->AddFilter(Ctx, where)) {
return nullptr;
}
@@ -3032,9 +3032,9 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet
}
TNodePtr having;
- if (node.HasBlock12()) {
+ if (node.HasBlock12()) {
TSqlExpression expr(Ctx, Mode);
- having = expr.Build(node.GetBlock12().GetRule_expr2());
+ having = expr.Build(node.GetBlock12().GetRule_expr2());
if (!having) {
return nullptr;
}
@@ -3043,7 +3043,7 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet
PushNamedNode(TArgPlaceholderNode::ProcessRow, BuildColumn(Ctx.Pos(), "*"));
TSqlCallExpr call(Ctx, Mode);
- bool initRet = call.Init(node.GetRule_call_expr9());
+ bool initRet = call.Init(node.GetRule_call_expr9());
if (initRet) {
call.IncCounters();
}
@@ -3062,11 +3062,11 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet
return {};
}
- if (node.HasBlock10()) {
- udf->SetLabel(IdOrString(node.GetBlock10().GetRule_id_or_string2(), *this));
+ if (node.HasBlock10()) {
+ udf->SetLabel(IdOrString(node.GetBlock10().GetRule_id_or_string2(), *this));
}
- const auto reduceMode = node.HasBlock8() ? ReduceMode::ByAll : ReduceMode::ByPartition;
+ const auto reduceMode = node.HasBlock8() ? ReduceMode::ByAll : ReduceMode::ByPartition;
return BuildReduce(startPos, reduceMode, std::move(source), std::move(orderBy), std::move(keys), std::move(args), udf, having, settings);
}
@@ -3192,7 +3192,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
if (!ValidateSelectColumns(terms)) {
return nullptr;
}
- return BuildSelectCore(Ctx, startPos, std::move(source), groupByExpr, groupBy, orderBy, having,
+ return BuildSelectCore(Ctx, startPos, std::move(source), groupByExpr, groupBy, orderBy, having,
std::move(windowSpec), hoppingWindowSpec, std::move(terms), distinct, std::move(without), stream, settings);
}
@@ -3837,7 +3837,7 @@ TSourcePtr TSqlSelect::Build(const TRule_select_stmt& node, TPosition& selectPos
return nullptr;
}
- TPosition unionPos = selectPos; // Position of first select
+ TPosition unionPos = selectPos; // Position of first select
TVector<TSourcePtr> sources;
sources.emplace_back(std::move(res));
for (auto& b: node.GetBlock2()) {
@@ -3867,7 +3867,7 @@ TSourcePtr TSqlSelect::Build(const TRule_select_stmt& node, TPosition& selectPos
return std::move(sources[0]);
}
- res = BuildUnionAll(unionPos, std::move(sources));
+ res = BuildUnionAll(unionPos, std::move(sources));
return res;
}
@@ -4150,7 +4150,7 @@ bool TSqlIntoTable::ValidateServiceName(const TRule_into_table_stmt& node, const
if (!table.Check(Ctx)) {
return false;
}
- auto serviceName = to_lower(table.ServiceName(Ctx));
+ auto serviceName = to_lower(table.ServiceName(Ctx));
const bool isMapReduce = serviceName == YtProviderName;
const bool isKikimr = serviceName == KikimrProviderName;
const bool isRtmr = serviceName == RtmrProviderName;
@@ -4213,7 +4213,7 @@ private:
bool ExportStatement(const TRule_export_stmt& stmt);
bool AlterTableAddColumns(TVector<TNodePtr>& blocks, const TRule_alter_table_add_column& node, const TTableRef& tr);
bool AlterTableDropColumn(TVector<TNodePtr>& blocks, const TRule_alter_table_drop_column& node, const TTableRef& tr);
- TNodePtr PragmaStatement(const TRule_pragma_stmt& stmt, bool& success);
+ TNodePtr PragmaStatement(const TRule_pragma_stmt& stmt, bool& success);
void AddStatementToBlocks(TVector<TNodePtr>& blocks, TNodePtr node);
TNodePtr Build(const TRule_delete_stmt& stmt);
@@ -4273,14 +4273,14 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
}
switch (altCase) {
case TRule_sql_stmt_core::kAltSqlStmtCore1: {
- bool success = false;
- TNodePtr nodeExpr = PragmaStatement(core.GetAlt_sql_stmt_core1().GetRule_pragma_stmt1(), success);
- if (!success) {
+ bool success = false;
+ TNodePtr nodeExpr = PragmaStatement(core.GetAlt_sql_stmt_core1().GetRule_pragma_stmt1(), success);
+ if (!success) {
return false;
}
- if (nodeExpr) {
+ if (nodeExpr) {
AddStatementToBlocks(blocks, nodeExpr);
- }
+ }
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore2: {
@@ -4297,38 +4297,38 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
}
case TRule_sql_stmt_core::kAltSqlStmtCore3: {
Ctx.BodyPart();
- TVector<TString> names;
- auto nodeExpr = NamedNode(core.GetAlt_sql_stmt_core3().GetRule_named_nodes_stmt1(), names);
+ TVector<TString> names;
+ auto nodeExpr = NamedNode(core.GetAlt_sql_stmt_core3().GetRule_named_nodes_stmt1(), names);
if (!nodeExpr) {
return false;
}
- TVector<TNodePtr> nodes;
+ TVector<TNodePtr> nodes;
auto subquery = nodeExpr->GetSource();
if (subquery) {
const auto alias = Ctx.MakeName("subquerynode");
const auto ref = Ctx.MakeName("subquery");
blocks.push_back(BuildSubquery(subquery, alias, Mode == NSQLTranslation::ESqlMode::SUBQUERY, names.size() == 1 ? -1 : names.size()));
blocks.back()->SetLabel(ref);
-
- for (size_t i = 0; i < names.size(); ++i) {
- nodes.push_back(BuildSubqueryRef(blocks.back(), ref, names.size() == 1 ? -1 : i));
- }
- } else {
- if (names.size() > 1) {
- const auto ref = Ctx.MakeName("tie");
- blocks.push_back(BuildTupleResult(nodeExpr, names.size()));
- blocks.back()->SetLabel(ref);
- for (size_t i = 0; i < names.size(); ++i) {
- nodes.push_back(nodeExpr->Y("Nth", ref, nodeExpr->Q(ToString(i))));
- }
- } else {
- nodes.push_back(std::move(nodeExpr));
- }
- }
-
- for (size_t i = 0; i < names.size(); ++i) {
+
+ for (size_t i = 0; i < names.size(); ++i) {
+ nodes.push_back(BuildSubqueryRef(blocks.back(), ref, names.size() == 1 ? -1 : i));
+ }
+ } else {
+ if (names.size() > 1) {
+ const auto ref = Ctx.MakeName("tie");
+ blocks.push_back(BuildTupleResult(nodeExpr, names.size()));
+ blocks.back()->SetLabel(ref);
+ for (size_t i = 0; i < names.size(); ++i) {
+ nodes.push_back(nodeExpr->Y("Nth", ref, nodeExpr->Q(ToString(i))));
+ }
+ } else {
+ nodes.push_back(std::move(nodeExpr));
+ }
+ }
+
+ for (size_t i = 0; i < names.size(); ++i) {
PushNamedNode(names[i], nodes[i]);
- }
+ }
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore4: {
@@ -4566,7 +4566,7 @@ bool TSqlQuery::AlterTableDropColumn(TVector<TNodePtr>& blocks, const TRule_alte
return true;
}
-TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success) {
+TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success) {
success = false;
const TString& prefix = OptIdPrefixAsStr(stmt.GetRule_opt_id_prefix2(), *this);
const TString& lowerPrefix = to_lower(prefix);
@@ -4576,12 +4576,12 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (!normalizeError.Empty()) {
Error() << normalizeError->Message;
Ctx.IncrementMonCounter("sql_errors", "NormalizePragmaError");
- return {};
+ return {};
}
TVector<TDeferredAtom> values;
TVector<const TRule_pragma_value*> pragmaValues;
- bool pragmaValueDefault = false;
+ bool pragmaValueDefault = false;
if (stmt.GetBlock4().HasAlt1()) {
pragmaValues.push_back(&stmt.GetBlock4().GetAlt1().GetRule_pragma_value2());
}
@@ -4597,13 +4597,13 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
for (auto pragmaValue : pragmaValues) {
if (pragmaValue->HasAlt_pragma_value3()) {
values.push_back(TDeferredAtom(Ctx.Pos(), StringContent(Ctx, pragmaValue->GetAlt_pragma_value3().GetToken1().GetValue())));
- }
- else if (pragmaValue->HasAlt_pragma_value2()
- && pragmaValue->GetAlt_pragma_value2().GetRule_id1().HasAlt_id2()
+ }
+ else if (pragmaValue->HasAlt_pragma_value2()
+ && pragmaValue->GetAlt_pragma_value2().GetRule_id1().HasAlt_id2()
&& "default" == Id(pragmaValue->GetAlt_pragma_value2().GetRule_id1(), *this))
- {
- pragmaValueDefault = true;
- }
+ {
+ pragmaValueDefault = true;
+ }
else if (withConfigure && pragmaValue->HasAlt_pragma_value5()) {
auto bindName = NamedNodeImpl(pragmaValue->GetAlt_pragma_value5().GetRule_bind_parameter1(), *this);
auto namedNode = GetNamedNode(bindName);
@@ -4617,7 +4617,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
} else {
Error() << "Expected string" << (withConfigure ? ", named parameter" : "") << " or 'default' keyword as pragma value for pragma: " << pragma;
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
}
@@ -4628,7 +4628,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
return{};
}
- if (normalizedPragma == "refselect") {
+ if (normalizedPragma == "refselect") {
Ctx.PragmaRefSelect = true;
Ctx.IncrementMonCounter("sql_pragma", "RefSelect");
} else if (normalizedPragma == "sampleselect") {
@@ -4641,7 +4641,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (values.size() != 1 || pragmaValueDefault) {
Error() << "Expected file alias as pragma value";
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
Ctx.IncrementMonCounter("sql_pragma", "udf");
@@ -4651,7 +4651,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (values.size() != 2U || pragmaValueDefault) {
Error() << "Expected file alias and url as pragma values";
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
Ctx.IncrementMonCounter("sql_pragma", "file");
@@ -4679,9 +4679,9 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_INFERSCHEME) << "PRAGMA InferScheme is deprecated, please use PRAGMA yt.InferSchema instead.";
Ctx.PragmaInferSchema = true;
Ctx.IncrementMonCounter("sql_pragma", "InferSchema");
- } else if (normalizedPragma == "directread") {
- Ctx.PragmaDirectRead = true;
- Ctx.IncrementMonCounter("sql_pragma", "DirectRead");
+ } else if (normalizedPragma == "directread") {
+ Ctx.PragmaDirectRead = true;
+ Ctx.IncrementMonCounter("sql_pragma", "DirectRead");
} else if (normalizedPragma == "equijoin") {
Ctx.IncrementMonCounter("sql_pragma", "EquiJoin");
} else if (normalizedPragma == "autocommit") {
@@ -4690,17 +4690,17 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
} else if (normalizedPragma == "tablepathprefix") {
if (values.size() == 1) {
if (!Ctx.SetPathPrefix(*values[0].GetLiteral())) {
- return {};
+ return {};
}
} else if (values.size() == 2) {
if (!Ctx.SetPathPrefix(*values[1].GetLiteral(), *values[0].GetLiteral())) {
- return {};
+ return {};
}
} else {
Error() << "Expected path prefix or tuple of (Provider, PathPrefix) or"
<< " (Cluster, PathPrefix) as pragma value";
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
Ctx.IncrementMonCounter("sql_pragma", "PathPrefix");
@@ -4708,14 +4708,14 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (values.size() != 1 || !TryFromString(*values[0].GetLiteral(), Ctx.PragmaGroupByLimit)) {
Error() << "Expected single unsigned integer argument for: " << pragma;
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
Ctx.IncrementMonCounter("sql_pragma", "GroupByLimit");
} else if (normalizedPragma == "groupbycubelimit") {
if (values.size() != 1 || !TryFromString(*values[0].GetLiteral(), Ctx.PragmaGroupByCubeLimit)) {
Error() << "Expected single unsigned integer argument for: " << pragma;
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
Ctx.IncrementMonCounter("sql_pragma", "GroupByCubeLimit");
}
@@ -4806,9 +4806,9 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
return {};
}
Ctx.IncrementMonCounter("sql_pragma", "ClassicDivision");
- } else if (normalizedPragma == "disableunordered") {
- Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_PRAGMA)
- << "Use of deprecated DisableUnordered pragma. It will be dropped soon";
+ } else if (normalizedPragma == "disableunordered") {
+ Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_PRAGMA)
+ << "Use of deprecated DisableUnordered pragma. It will be dropped soon";
} else if (normalizedPragma == "pullupflatmapoverjoin") {
Ctx.PragmaPullUpFlatMapOverJoin = true;
Ctx.IncrementMonCounter("sql_pragma", "PullUpFlatMapOverJoin");
@@ -4825,7 +4825,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
} else {
Error() << "Unknown pragma: " << pragma;
Ctx.IncrementMonCounter("sql_errors", "UnknownPragma");
- return {};
+ return {};
}
} else {
if (lowerPrefix == "yson") {
@@ -4857,7 +4857,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
Error() << "Unknown pragma prefix: " << prefix << ", please use cluster name or one of provider " <<
JoinRange(", ", Providers.cbegin(), Providers.cend());
Ctx.IncrementMonCounter("sql_errors", "UnknownPragma");
- return {};
+ return {};
}
}
@@ -4865,22 +4865,22 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (values.size() > 1) {
Error() << "Expected at most one value in the pragma";
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
} else {
- if (pragmaValueDefault || values.size() < 1) {
+ if (pragmaValueDefault || values.size() < 1) {
Error() << "Expected at least one value in the pragma";
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
}
success = true;
Ctx.IncrementMonCounter("sql_pragma", pragma);
- return BuildPragma(Ctx.Pos(), lowerPrefix, normalizedPragma, values, pragmaValueDefault);
+ return BuildPragma(Ctx.Pos(), lowerPrefix, normalizedPragma, values, pragmaValueDefault);
}
success = true;
- return {};
+ return {};
}
TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) {
@@ -4889,7 +4889,7 @@ TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) {
return nullptr;
}
- auto serviceName = to_lower(table.ServiceName(Ctx));
+ auto serviceName = to_lower(table.ServiceName(Ctx));
const bool isKikimr = serviceName == KikimrProviderName;
if (!isKikimr) {
@@ -4938,7 +4938,7 @@ TNodePtr TSqlQuery::Build(const TRule_update_stmt& stmt) {
return nullptr;
}
- auto serviceName = to_lower(table.ServiceName(Ctx));
+ auto serviceName = to_lower(table.ServiceName(Ctx));
const bool isKikimr = serviceName == KikimrProviderName;
if (!isKikimr) {
diff --git a/ydb/library/yql/sql/v0/sql.h b/ydb/library/yql/sql/v0/sql.h
index fda4387c27..bc32041d4a 100644
--- a/ydb/library/yql/sql/v0/sql.h
+++ b/ydb/library/yql/sql/v0/sql.h
@@ -5,9 +5,9 @@
#include <ydb/library/yql/public/issue/yql_warning.h>
#include <ydb/library/yql/public/issue/yql_issue_manager.h>
#include <ydb/library/yql/sql/settings/translation_settings.h>
-
+
#include <google/protobuf/message.h>
-
+
namespace NSQLTranslationV0 {
NYql::TAstParseResult SqlToYql(const TString& query, const NSQLTranslation::TTranslationSettings& settings, NYql::TWarningRules* warningRules = nullptr);
diff --git a/ydb/library/yql/sql/v0/sql_ut.cpp b/ydb/library/yql/sql/v0/sql_ut.cpp
index 3a70aef93c..802170ea1f 100644
--- a/ydb/library/yql/sql/v0/sql_ut.cpp
+++ b/ydb/library/yql/sql/v0/sql_ut.cpp
@@ -26,7 +26,7 @@ TString Err2Str(NYql::TAstParseResult& res, EDebugOutput debug = EDebugOutput::N
NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ESqlMode mode = NSQLTranslation::ESqlMode::QUERY, size_t maxErrors = 10, const TString& provider = {}, EDebugOutput debug = EDebugOutput::None) {
google::protobuf::Arena arena;
- const auto service = provider ? provider : TString(NYql::YtProviderName);
+ const auto service = provider ? provider : TString(NYql::YtProviderName);
const TString cluster = "plato";
NSQLTranslation::TTranslationSettings settings;
settings.ClusterMapping[cluster] = service;
@@ -786,7 +786,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
TWordCountHive elementStat = {{TString("Kikimr.PushData"), 0}};
VerifyProgram(res, elementStat, verifyLine);
- UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]);
+ UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]);
}
Y_UNIT_TEST(ProcessUserTypeAuth) {
@@ -906,7 +906,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) {
UNIT_ASSERT(!res.Root);
TString a1 = Err2Str(res);
- TString a2(R"foo(<main>:1:14: Error: Unexpected character 'с' (Unicode character <1089>) : cannot match to any predicted input...
+ TString a2(R"foo(<main>:1:14: Error: Unexpected character 'с' (Unicode character <1089>) : cannot match to any predicted input...
<main>:1:15: Error: Unexpected character : cannot match to any predicted input...
diff --git a/ydb/library/yql/sql/v1/SQLv1.g.in b/ydb/library/yql/sql/v1/SQLv1.g.in
index ef394c7a00..7cc3322876 100644
--- a/ydb/library/yql/sql/v1/SQLv1.g.in
+++ b/ydb/library/yql/sql/v1/SQLv1.g.in
@@ -343,29 +343,29 @@ select_kind: (DISCARD)? (process_core | reduce_core | select_core) (INTO RESULT
process_core:
PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING using_call_expr (AS an_id)?
(WITH external_call_settings)?
- (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?)?
+ (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?)?
;
external_call_param: an_id EQUALS expr;
external_call_settings: external_call_param (COMMA external_call_param)*;
reduce_core:
- REDUCE named_single_source (COMMA named_single_source)* (PRESORT sort_specification_list)?
+ REDUCE named_single_source (COMMA named_single_source)* (PRESORT sort_specification_list)?
ON column_list USING ALL? using_call_expr (AS an_id)?
- (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?
+ (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?
;
opt_set_quantifier: (ALL | DISTINCT)?;
select_core:
(FROM join_source)? SELECT STREAM? opt_set_quantifier result_column (COMMA result_column)* COMMA? (WITHOUT without_column_list)? (FROM join_source)? (WHERE expr)?
- group_by_clause? (HAVING expr)? window_clause? ext_order_by_clause?
+ group_by_clause? (HAVING expr)? window_clause? ext_order_by_clause?
;
order_by_clause: ORDER BY sort_specification_list;
-ext_order_by_clause: ASSUME? order_by_clause;
-
+ext_order_by_clause: ASSUME? order_by_clause;
+
group_by_clause: GROUP COMPACT? BY opt_set_quantifier grouping_element_list;
grouping_element_list: grouping_element (COMMA grouping_element)*;
@@ -1124,7 +1124,7 @@ ANY: A N Y;
ARRAY: A R R A Y;
AS: A S;
ASC: A S C;
-ASSUME: A S S U M E;
+ASSUME: A S S U M E;
ASYNC: A S Y N C;
ATTACH: A T T A C H;
AUTOINCREMENT: A U T O I N C R E M E N T;
diff --git a/ydb/library/yql/sql/v1/builtin.cpp b/ydb/library/yql/sql/v1/builtin.cpp
index 43ef1243f1..9a37aabf94 100644
--- a/ydb/library/yql/sql/v1/builtin.cpp
+++ b/ydb/library/yql/sql/v1/builtin.cpp
@@ -470,7 +470,7 @@ public:
"Please consult https://yql.yandex-team.ru/docs/yt/builtins/basic/#tablepath for possible workaround";
}
- Args.push_back(Y("TablePath", Y("DependsOn", "row")));
+ Args.push_back(Y("TablePath", Y("DependsOn", "row")));
}
if (Args.size() == 2) {
@@ -1635,7 +1635,7 @@ public:
Node = Y("block", Q(L(block, Y("return", "res"))));
} else {
- Node = ctx.EnableSystemColumns ? Y("RemoveSystemMembers", "row") : BuildAtom(Pos, "row", 0);
+ Node = ctx.EnableSystemColumns ? Y("RemoveSystemMembers", "row") : BuildAtom(Pos, "row", 0);
}
return true;
}
@@ -1663,24 +1663,24 @@ TTableRows::TTableRows(TPosition pos, const TVector<TNodePtr>& args)
{}
TTableRows::TTableRows(TPosition pos, ui32 argsCount)
- : INode(pos)
+ : INode(pos)
, ArgsCount(argsCount)
{}
-bool TTableRows::DoInit(TContext& ctx, ISource* /*src*/) {
+bool TTableRows::DoInit(TContext& ctx, ISource* /*src*/) {
if (ArgsCount > 0) {
ctx.Error(Pos) << "TableRows requires exactly 0 arguments";
return false;
}
- Node = ctx.EnableSystemColumns ? Y("RemoveSystemMembers", "inputRowsList") : BuildAtom(Pos, "inputRowsList", 0);
+ Node = ctx.EnableSystemColumns ? Y("RemoveSystemMembers", "inputRowsList") : BuildAtom(Pos, "inputRowsList", 0);
return true;
}
-TAstNode* TTableRows::Translate(TContext& ctx) const {
- Y_VERIFY_DEBUG(Node);
- return Node->Translate(ctx);
-}
-
+TAstNode* TTableRows::Translate(TContext& ctx) const {
+ Y_VERIFY_DEBUG(Node);
+ return Node->Translate(ctx);
+}
+
void TTableRows::DoUpdateState() const {
State.Set(ENodeState::Const, false);
}
@@ -2561,7 +2561,7 @@ struct TBuiltinFuncData {
{"dictpayloads", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("DictPayloads", 1, 1) },
{"dictitems", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("DictItems", 1, 1) },
{"dictlookup", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("Lookup", 2, 2) },
- {"dictcontains", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("Contains", 2, 2) },
+ {"dictcontains", BuildNamedArgcBuiltinFactoryCallback<TCallNodeImpl>("Contains", 2, 2) },
// Atom builtins
{"asatom", BuildSimpleBuiltinFactoryCallback<TYqlAsAtom>()},
diff --git a/ydb/library/yql/sql/v1/context.cpp b/ydb/library/yql/sql/v1/context.cpp
index c08fe96cae..b529acc753 100644
--- a/ydb/library/yql/sql/v1/context.cpp
+++ b/ydb/library/yql/sql/v1/context.cpp
@@ -39,9 +39,9 @@ TNodePtr AddTablePathPrefix(TContext& ctx, TStringBuf prefixPath, const TDeferre
return result.Build();
}
-typedef bool TContext::*TPragmaField;
-
-THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = {
+typedef bool TContext::*TPragmaField;
+
+THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = {
{"AnsiOptionalAs", &TContext::AnsiOptionalAs},
{"WarnOnAnsiAliasShadowing", &TContext::WarnOnAnsiAliasShadowing},
{"PullUpFlatMapOverJoin", &TContext::PragmaPullUpFlatMapOverJoin},
@@ -53,8 +53,8 @@ THashMap<TStringBuf, TPragmaField> CTX_PRAGMA_FIELDS = {
{"CoalesceJoinKeysOnQualifiedAll", &TContext::CoalesceJoinKeysOnQualifiedAll},
{"UnorderedSubqueries", &TContext::UnorderedSubqueries},
{"FlexibleTypes", &TContext::FlexibleTypes},
-};
-
+};
+
typedef TMaybe<bool> TContext::*TPragmaMaybeField;
THashMap<TStringBuf, TPragmaMaybeField> CTX_PRAGMA_MAYBE_FIELDS = {
@@ -93,23 +93,23 @@ TContext::TContext(const NSQLTranslation::TTranslationSettings& settings,
Position.File = settings.File;
- for (auto& flag: settings.Flags) {
- bool value = true;
- TStringBuf key = flag;
- auto ptr = CTX_PRAGMA_FIELDS.FindPtr(key);
+ for (auto& flag: settings.Flags) {
+ bool value = true;
+ TStringBuf key = flag;
+ auto ptr = CTX_PRAGMA_FIELDS.FindPtr(key);
auto ptrMaybe = CTX_PRAGMA_MAYBE_FIELDS.FindPtr(key);
if (!ptr && !ptrMaybe && key.SkipPrefix("Disable")) {
- value = false;
- ptr = CTX_PRAGMA_FIELDS.FindPtr(key);
+ value = false;
+ ptr = CTX_PRAGMA_FIELDS.FindPtr(key);
ptrMaybe = CTX_PRAGMA_MAYBE_FIELDS.FindPtr(key);
- }
+ }
if (ptr) {
this->*(*ptr) = value;
} else if (ptrMaybe) {
this->*(*ptrMaybe) = value;
}
- }
- DiscoveryMode = (NSQLTranslation::ESqlMode::DISCOVERY == Settings.Mode);
+ }
+ DiscoveryMode = (NSQLTranslation::ESqlMode::DISCOVERY == Settings.Mode);
}
TContext::~TContext()
@@ -134,13 +134,13 @@ TString TContext::MakeName(const TString& name) {
return str;
}
-IOutputStream& TContext::Error(NYql::TIssueCode code) {
- return Error(Pos(), code);
+IOutputStream& TContext::Error(NYql::TIssueCode code) {
+ return Error(Pos(), code);
}
-IOutputStream& TContext::Error(NYql::TPosition pos, NYql::TIssueCode code) {
+IOutputStream& TContext::Error(NYql::TPosition pos, NYql::TIssueCode code) {
HasPendingErrors = true;
- return MakeIssue(TSeverityIds::S_ERROR, code, pos);
+ return MakeIssue(TSeverityIds::S_ERROR, code, pos);
}
IOutputStream& TContext::Warning(NYql::TPosition pos, NYql::TIssueCode code) {
@@ -200,10 +200,10 @@ IOutputStream& TContext::MakeIssue(ESeverity severity, TIssueCode code, NYql::TP
bool TContext::SetPathPrefix(const TString& value, TMaybe<TString> arg) {
if (arg.Defined()) {
- if (*arg == YtProviderName
- || *arg == KikimrProviderName
- || *arg == RtmrProviderName
- )
+ if (*arg == YtProviderName
+ || *arg == KikimrProviderName
+ || *arg == RtmrProviderName
+ )
{
ProviderPathPrefixes[*arg] = value;
return true;
@@ -347,33 +347,33 @@ TNodePtr TScopedState::LookupNode(const TString& name) {
}
bool TContext::HasNonYtProvider(const ISource& source) const {
- TTableList tableList;
- source.GetInputTables(tableList);
+ TTableList tableList;
+ source.GetInputTables(tableList);
- TSet<TString> clusters;
- for (auto& it: tableList) {
+ TSet<TString> clusters;
+ for (auto& it: tableList) {
if (it.Service != YtProviderName) {
return true;
- }
- }
-
+ }
+ }
+
for (auto& cl: Scoped->Local.UsedClusters) {
if (cl.first != YtProviderName) {
return true;
- }
- }
-
+ }
+ }
+
return false;
-}
-
-bool TContext::UseUnordered(const ISource& source) const {
- return !HasNonYtProvider(source);
-}
-
-bool TContext::UseUnordered(const TTableRef& table) const {
- return YtProviderName == table.Service;
-}
-
+}
+
+bool TContext::UseUnordered(const ISource& source) const {
+ return !HasNonYtProvider(source);
+}
+
+bool TContext::UseUnordered(const TTableRef& table) const {
+ return YtProviderName == table.Service;
+}
+
TMaybe<EColumnRefState> GetFunctionArgColumnStatus(TContext& ctx, const TString& module, const TString& func, size_t argIndex) {
static const TSet<TStringBuf> denyForAllArgs = {
diff --git a/ydb/library/yql/sql/v1/context.h b/ydb/library/yql/sql/v1/context.h
index 8951757486..6a03c35256 100644
--- a/ydb/library/yql/sql/v1/context.h
+++ b/ydb/library/yql/sql/v1/context.h
@@ -88,8 +88,8 @@ namespace NSQLTranslationV1 {
void ClearBlockScope();
TString MakeName(const TString& name);
- IOutputStream& Error(NYql::TIssueCode code = NYql::TIssuesIds::DEFAULT_ERROR);
- IOutputStream& Error(NYql::TPosition pos, NYql::TIssueCode code = NYql::TIssuesIds::DEFAULT_ERROR);
+ IOutputStream& Error(NYql::TIssueCode code = NYql::TIssuesIds::DEFAULT_ERROR);
+ IOutputStream& Error(NYql::TPosition pos, NYql::TIssueCode code = NYql::TIssuesIds::DEFAULT_ERROR);
IOutputStream& Warning(NYql::TPosition pos, NYql::TIssueCode code);
IOutputStream& Info(NYql::TPosition pos);
@@ -139,9 +139,9 @@ namespace NSQLTranslationV1 {
}
bool HasNonYtProvider(const ISource& source) const;
- bool UseUnordered(const ISource& source) const;
- bool UseUnordered(const TTableRef& table) const;
-
+ bool UseUnordered(const ISource& source) const;
+ bool UseUnordered(const TTableRef& table) const;
+
bool SetPathPrefix(const TString& value, TMaybe<TString> arg = TMaybe<TString>());
TNodePtr GetPrefixedPath(const TString& service, const TDeferredAtom& cluster, const TDeferredAtom& path);
@@ -225,14 +225,14 @@ namespace NSQLTranslationV1 {
bool PragmaAutoCommit = false;
bool SimpleColumns = true;
bool CoalesceJoinKeysOnQualifiedAll = false;
- bool PragmaDirectRead = false;
+ bool PragmaDirectRead = false;
bool PragmaYsonFast = true;
bool PragmaYsonAutoConvert = false;
bool PragmaYsonStrict = true;
bool PragmaRegexUseRe2 = true;
bool PragmaPullUpFlatMapOverJoin = true;
bool WarnUnnamedColumns = false;
- bool DiscoveryMode = false;
+ bool DiscoveryMode = false;
bool EnableSystemColumns = true;
bool DqEngineEnable = false;
bool DqEngineForce = false;
diff --git a/ydb/library/yql/sql/v1/insert.cpp b/ydb/library/yql/sql/v1/insert.cpp
index 4f4c863018..98f32ae5d2 100644
--- a/ydb/library/yql/sql/v1/insert.cpp
+++ b/ydb/library/yql/sql/v1/insert.cpp
@@ -46,7 +46,7 @@ public:
}
TNodePtr BuildFilter(TContext& ctx, const TString& label) override {
- Y_UNUSED(ctx);
+ Y_UNUSED(ctx);
Y_UNUSED(label);
return nullptr;
}
@@ -240,10 +240,10 @@ public:
return new TModifyBySource(Pos, OperationHumanName, ColumnsHint, Source->CloneSource());
}
- EOrderKind GetOrderKind() const final {
- return Source->GetOrderKind();
- }
-
+ EOrderKind GetOrderKind() const final {
+ return Source->GetOrderKind();
+ }
+
private:
TString OperationHumanName;
TSourcePtr Source;
@@ -303,7 +303,7 @@ public:
options = L(options, Q(Y(Q("filter"), TableSource->BuildFilterLambda())));
}
- bool unordered = false;
+ bool unordered = false;
if (Values) {
if (!Values->Init(ctx, TableSource.Get())) {
return false;
@@ -315,7 +315,7 @@ public:
if (!values) {
return false;
}
- unordered = (EOrderKind::None == Values->GetOrderKind());
+ unordered = (EOrderKind::None == Values->GetOrderKind());
}
TNodePtr node(BuildInputTables(Pos, tableList, false, Scoped));
@@ -336,9 +336,9 @@ public:
}
if (values) {
node = L(node, Y("let", "values", values));
- if (unordered && ctx.UseUnordered(Table)) {
- node = L(node, Y("let", "values", Y("Unordered", "values")));
- }
+ if (unordered && ctx.UseUnordered(Table)) {
+ node = L(node, Y("let", "values", Y("Unordered", "values")));
+ }
} else {
node = L(node, Y("let", "values", Y("Void")));
}
diff --git a/ydb/library/yql/sql/v1/join.cpp b/ydb/library/yql/sql/v1/join.cpp
index 29f5b3ec4b..461b8ac4b6 100644
--- a/ydb/library/yql/sql/v1/join.cpp
+++ b/ydb/library/yql/sql/v1/join.cpp
@@ -520,15 +520,15 @@ public:
}
TNodePtr equiJoin(Y("EquiJoin"));
- bool ordered = false;
+ bool ordered = false;
for (size_t i = 0; i < Sources.size(); ++i) {
auto& source = Sources[i];
auto sourceNode = source->Build(ctx);
if (!sourceNode) {
return nullptr;
}
- const bool useOrderedForSource = ctx.UseUnordered(*source);
- ordered = ordered || useOrderedForSource;
+ const bool useOrderedForSource = ctx.UseUnordered(*source);
+ ordered = ordered || useOrderedForSource;
if (source->IsFlattenByColumns() || source->IsFlattenColumns()) {
auto flatten = source->IsFlattenByColumns() ?
source->BuildFlattenByColumns("row") :
@@ -548,7 +548,7 @@ public:
block = L(block, Y("let", "flatten", Y(useOrderedForSource ? "OrderedFlatMap" : "FlatMap", "flatten", BuildLambda(Pos, Y("row"), premap))));
}
- block = L(block, Y("let", "flatten", Y(useOrderedForSource ? "OrderedFlatMap" : "FlatMap", "flatten", BuildLambda(Pos, Y("row"), flatten, "res"))));
+ block = L(block, Y("let", "flatten", Y(useOrderedForSource ? "OrderedFlatMap" : "FlatMap", "flatten", BuildLambda(Pos, Y("row"), flatten, "res"))));
sourceNode = Y("block", Q(L(block, Y("return", "flatten"))));
}
TNodePtr extraMembers;
@@ -565,7 +565,7 @@ public:
);
}
if (extraMembers) {
- sourceNode = Y(useOrderedForSource ? "OrderedMap" : "Map", sourceNode, BuildLambda(Pos, Y("row"), extraMembers, "row"));
+ sourceNode = Y(useOrderedForSource ? "OrderedMap" : "Map", sourceNode, BuildLambda(Pos, Y("row"), extraMembers, "row"));
}
if (ctx.EnableSystemColumns && source->IsTableSource()) {
sourceNode = Y("RemoveSystemMembers", sourceNode);
@@ -588,7 +588,7 @@ public:
}
equiJoin = L(equiJoin, joinTree, Q(options));
if (removeMembers) {
- equiJoin = Y(ordered ? "OrderedMap" : "Map", equiJoin, BuildLambda(Pos, Y("row"), removeMembers, "row"));
+ equiJoin = Y(ordered ? "OrderedMap" : "Map", equiJoin, BuildLambda(Pos, Y("row"), removeMembers, "row"));
}
return equiJoin;
}
diff --git a/ydb/library/yql/sql/v1/list_builtin.cpp b/ydb/library/yql/sql/v1/list_builtin.cpp
index bc4edc2e4c..032052e9e8 100644
--- a/ydb/library/yql/sql/v1/list_builtin.cpp
+++ b/ydb/library/yql/sql/v1/list_builtin.cpp
@@ -51,7 +51,7 @@ bool TListExtractBuiltin::DoInit(TContext& ctx, ISource* src) {
bool TListProcessBuiltin::CheckArgs(TContext& ctx, ISource* src) {
if (Args.size() != 2 ) {
- ctx.Error(Pos) << OpName << " requires exactly two parameters";
+ ctx.Error(Pos) << OpName << " requires exactly two parameters";
return false;
}
diff --git a/ydb/library/yql/sql/v1/node.cpp b/ydb/library/yql/sql/v1/node.cpp
index 870bbd9cc7..54dd17b1b8 100644
--- a/ydb/library/yql/sql/v1/node.cpp
+++ b/ydb/library/yql/sql/v1/node.cpp
@@ -684,11 +684,11 @@ bool TCallDirectRow::DoInit(TContext& ctx, ISource* src) {
ctx.Error(Pos) << "Failed to use function: " << OpName << " with aggregation, join, flatten by or window functions";
return false;
}
- if (!TCallNode::DoInit(ctx, src)) {
- return false;
- }
- Nodes.push_back(Y("DependsOn", "row"));
- return true;
+ if (!TCallNode::DoInit(ctx, src)) {
+ return false;
+ }
+ Nodes.push_back(Y("DependsOn", "row"));
+ return true;
}
void TCallDirectRow::DoUpdateState() const {
@@ -1585,10 +1585,10 @@ bool ISource::IsStream() const {
return false;
}
-EOrderKind ISource::GetOrderKind() const {
- return EOrderKind::None;
-}
-
+EOrderKind ISource::GetOrderKind() const {
+ return EOrderKind::None;
+}
+
TWriteSettings ISource::GetWriteSettings() const {
return {};
}
@@ -1711,7 +1711,7 @@ TNodePtr ISource::BuildPrewindowMap(TContext& ctx) {
feed = Y("AddMember", feed, Q(name), exprNode);
}
}
- return Y(ctx.UseUnordered(*this) ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), Y("AsList", feed)));
+ return Y(ctx.UseUnordered(*this) ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), Y("AsList", feed)));
}
bool ISource::BuildSamplingLambda(TNodePtr& node) {
@@ -1987,12 +1987,12 @@ TNodePtr ISource::BuildSort(TContext& ctx, const TString& label) {
return nullptr;
}
-TNodePtr ISource::BuildCleanupColumns(TContext& ctx, const TString& label) {
- Y_UNUSED(ctx);
- Y_UNUSED(label);
- return nullptr;
-}
-
+TNodePtr ISource::BuildCleanupColumns(TContext& ctx, const TString& label) {
+ Y_UNUSED(ctx);
+ Y_UNUSED(label);
+ return nullptr;
+}
+
IJoin* ISource::GetJoin() {
return nullptr;
}
@@ -2082,8 +2082,8 @@ TNodePtr ISource::BuildSortSpec(const TVector<TSortSpecificationPtr>& orderBy, c
FillSortParts(orderBy, dirsNode, keySelectorNode);
if (traits) {
return Y("SortTraits", Y("TypeOf", label), dirsNode, keySelectorNode);
- } else if (assume) {
- return Y("AssumeSorted", label, dirsNode, keySelectorNode);
+ } else if (assume) {
+ return Y("AssumeSorted", label, dirsNode, keySelectorNode);
} else {
return Y("Sort", label, dirsNode, keySelectorNode);
}
@@ -3380,45 +3380,45 @@ TDeferredAtom MakeAtomFromExpression(TContext& ctx, TNodePtr node) {
return TDeferredAtom(wrappedNode, ctx);
}
-class TTupleResultNode: public INode {
-public:
- TTupleResultNode(TNodePtr&& tuple, int ensureTupleSize)
- : INode(tuple->GetPos())
- , Node(std::move(tuple))
- , EnsureTupleSize(ensureTupleSize)
- {
- }
-
- bool DoInit(TContext& ctx, ISource* src) override {
- if (!Node->Init(ctx, src)) {
- return false;
- }
-
- Node = Y("EnsureTupleSize", Node, Q(ToString(EnsureTupleSize)));
-
- return true;
- }
-
- TAstNode* Translate(TContext& ctx) const override {
- return Node->Translate(ctx);
- }
-
- TPtr DoClone() const final {
- return {};
- }
-
+class TTupleResultNode: public INode {
+public:
+ TTupleResultNode(TNodePtr&& tuple, int ensureTupleSize)
+ : INode(tuple->GetPos())
+ , Node(std::move(tuple))
+ , EnsureTupleSize(ensureTupleSize)
+ {
+ }
+
+ bool DoInit(TContext& ctx, ISource* src) override {
+ if (!Node->Init(ctx, src)) {
+ return false;
+ }
+
+ Node = Y("EnsureTupleSize", Node, Q(ToString(EnsureTupleSize)));
+
+ return true;
+ }
+
+ TAstNode* Translate(TContext& ctx) const override {
+ return Node->Translate(ctx);
+ }
+
+ TPtr DoClone() const final {
+ return {};
+ }
+
void DoVisitChildren(const TVisitFunc& func, TVisitNodeSet& visited) const final {
Y_VERIFY_DEBUG(Node);
Node->VisitTree(func, visited);
}
-protected:
- TNodePtr Node;
- const int EnsureTupleSize;
-};
-
-TNodePtr BuildTupleResult(TNodePtr tuple, int ensureTupleSize) {
- return new TTupleResultNode(std::move(tuple), ensureTupleSize);
-}
-
-
+protected:
+ TNodePtr Node;
+ const int EnsureTupleSize;
+};
+
+TNodePtr BuildTupleResult(TNodePtr tuple, int ensureTupleSize) {
+ return new TTupleResultNode(std::move(tuple), ensureTupleSize);
+}
+
+
} // namespace NSQLTranslationV1
diff --git a/ydb/library/yql/sql/v1/node.h b/ydb/library/yql/sql/v1/node.h
index de82b45a23..27dee3b310 100644
--- a/ydb/library/yql/sql/v1/node.h
+++ b/ydb/library/yql/sql/v1/node.h
@@ -798,13 +798,13 @@ namespace NSQLTranslationV1 {
ColumnOnly,
};
- enum class EOrderKind: int {
- None,
- Sort,
- Assume,
- Passthrough
- };
-
+ enum class EOrderKind: int {
+ None,
+ Sort,
+ Assume,
+ Passthrough
+ };
+
class IJoin;
class ISource: public INode {
public:
@@ -843,7 +843,7 @@ namespace NSQLTranslationV1 {
virtual bool IsCalcOverWindow() const;
virtual bool IsOverWindowSource() const;
virtual bool IsStream() const;
- virtual EOrderKind GetOrderKind() const;
+ virtual EOrderKind GetOrderKind() const;
virtual TWriteSettings GetWriteSettings() const;
virtual bool SetSamplingOptions(TContext& ctx, TPosition pos, ESampleMode mode, TNodePtr samplingRate, TNodePtr samplingSeed);
virtual bool SetTableHints(TContext& ctx, TPosition pos, const TTableHints& hints, const TTableHints& contextHints);
@@ -858,7 +858,7 @@ namespace NSQLTranslationV1 {
virtual TNodePtr BuildAggregation(const TString& label);
virtual TNodePtr BuildCalcOverWindow(TContext& ctx, const TString& label);
virtual TNodePtr BuildSort(TContext& ctx, const TString& label);
- virtual TNodePtr BuildCleanupColumns(TContext& ctx, const TString& label);
+ virtual TNodePtr BuildCleanupColumns(TContext& ctx, const TString& label);
virtual bool BuildSamplingLambda(TNodePtr& node);
virtual bool SetSamplingRate(TContext& ctx, TNodePtr samplingRate);
virtual IJoin* GetJoin();
@@ -1008,7 +1008,7 @@ namespace NSQLTranslationV1 {
TString View;
};
- class TTableRows final : public INode {
+ class TTableRows final : public INode {
public:
TTableRows(TPosition pos, const TVector<TNodePtr>& args);
TTableRows(TPosition pos, ui32 argsCount);
@@ -1018,11 +1018,11 @@ namespace NSQLTranslationV1 {
void DoUpdateState() const override;
TNodePtr DoClone() const final;
- TAstNode* Translate(TContext& ctx) const override;
+ TAstNode* Translate(TContext& ctx) const override;
private:
ui32 ArgsCount;
- TNodePtr Node;
+ TNodePtr Node;
};
class TSessionWindow final : public INode {
@@ -1218,7 +1218,7 @@ namespace NSQLTranslationV1 {
TNodePtr BuildYsonOptionsNode(TPosition pos, bool autoConvert, bool strict, bool fastYson);
TNodePtr BuildDoCall(TPosition pos, const TNodePtr& node);
- TNodePtr BuildTupleResult(TNodePtr tuple, int ensureTupleSize);
+ TNodePtr BuildTupleResult(TNodePtr tuple, int ensureTupleSize);
// Implemented in aggregation.cpp
TAggregationPtr BuildFactoryAggregation(TPosition pos, const TString& name, const TString& func, EAggregateMode aggMode, bool multi = false);
@@ -1257,9 +1257,9 @@ namespace NSQLTranslationV1 {
// Implemented in select.cpp
TNodePtr BuildSubquery(TSourcePtr source, const TString& alias, bool inSubquery, int ensureTupleSize, TScopedStatePtr scoped);
- TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleIndex = -1);
+ TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleIndex = -1);
TNodePtr BuildSourceNode(TPosition pos, TSourcePtr source, bool checkExist = false);
- TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources);
+ TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources);
TSourcePtr BuildFakeSource(TPosition pos, bool missingFrom = false);
TSourcePtr BuildNodeSource(TPosition pos, const TNodePtr& node, bool wrapToList = false);
TSourcePtr BuildTableSource(TPosition pos, const TTableRef& table, const TString& label = TString());
@@ -1279,7 +1279,7 @@ namespace NSQLTranslationV1 {
const TVector<TNodePtr>& groupByExpr,
const TVector<TNodePtr>& groupBy,
bool compactGroupBy,
- bool assumeSorted,
+ bool assumeSorted,
const TVector<TSortSpecificationPtr>& orderBy,
TNodePtr having,
TWinSpecs&& windowSpec,
@@ -1298,7 +1298,7 @@ namespace NSQLTranslationV1 {
ByAll,
};
TSourcePtr BuildReduce(TPosition pos, ReduceMode mode, TSourcePtr source, TVector<TSortSpecificationPtr>&& orderBy,
- TVector<TNodePtr>&& keys, TVector<TNodePtr>&& args, TNodePtr udf, TNodePtr having, const TWriteSettings& settings,
+ TVector<TNodePtr>&& keys, TVector<TNodePtr>&& args, TNodePtr udf, TNodePtr having, const TWriteSettings& settings,
const TVector<TSortSpecificationPtr>& assumeOrderBy, bool listCall);
TSourcePtr BuildProcess(TPosition pos, TSourcePtr source, TNodePtr with, bool withExtFunction, TVector<TNodePtr>&& terms, bool listCall,
bool prcessStream, const TWriteSettings& settings, const TVector<TSortSpecificationPtr>& assumeOrderBy);
diff --git a/ydb/library/yql/sql/v1/query.cpp b/ydb/library/yql/sql/v1/query.cpp
index 725356b9f5..0b5d9e1224 100644
--- a/ydb/library/yql/sql/v1/query.cpp
+++ b/ydb/library/yql/sql/v1/query.cpp
@@ -1,6 +1,6 @@
#include "node.h"
#include "context.h"
-
+
#include <ydb/library/yql/ast/yql_type_string.h>
#include <ydb/library/yql/core/yql_callable_names.h>
#include <ydb/library/yql/providers/common/provider/yql_provider_names.h>
@@ -8,8 +8,8 @@
#include <library/cpp/charset/ci_string.h>
-#include <util/digest/fnv.h>
-
+#include <util/digest/fnv.h>
+
using namespace NYql;
namespace NSQLTranslationV1 {
@@ -235,10 +235,10 @@ public:
ctx.Error(Pos) << Func << " requires from " << minArgs << " to " << maxArgs << " arguments, but got: " << Args.size();
return nullptr;
}
- if (ctx.DiscoveryMode) {
- ctx.Error(Pos, TIssuesIds::YQL_NOT_ALLOWED_IN_DISCOVERY) << Func << " is not allowed in Discovery mode";
- return nullptr;
- }
+ if (ctx.DiscoveryMode) {
+ ctx.Error(Pos, TIssuesIds::YQL_NOT_ALLOWED_IN_DISCOVERY) << Func << " is not allowed in Discovery mode";
+ return nullptr;
+ }
for (ui32 index=0; index < Args.size(); ++index) {
auto& arg = Args[index];
@@ -373,11 +373,11 @@ public:
return nullptr;
}
- if (ctx.DiscoveryMode) {
- ctx.Error(Pos, TIssuesIds::YQL_NOT_ALLOWED_IN_DISCOVERY) << Func << " is not allowed in Discovery mode";
- return nullptr;
- }
-
+ if (ctx.DiscoveryMode) {
+ ctx.Error(Pos, TIssuesIds::YQL_NOT_ALLOWED_IN_DISCOVERY) << Func << " is not allowed in Discovery mode";
+ return nullptr;
+ }
+
for (ui32 index = 0; index < Args.size(); ++index) {
auto& arg = Args[index];
if (arg.HasAt) {
@@ -1637,29 +1637,29 @@ public:
}
if (ctx.Settings.Mode != NSQLTranslation::ESqlMode::LIBRARY) {
- auto configSource = Y("DataSource", BuildQuotedAtom(Pos, TString(ConfigProviderName)));
- auto resultSink = Y("DataSink", BuildQuotedAtom(Pos, TString(ResultProviderName)));
+ auto configSource = Y("DataSource", BuildQuotedAtom(Pos, TString(ConfigProviderName)));
+ auto resultSink = Y("DataSink", BuildQuotedAtom(Pos, TString(ResultProviderName)));
for (const auto& warningPragma : ctx.WarningPolicy.GetRules()) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "Warning"), BuildQuotedAtom(Pos, warningPragma.GetPattern()),
BuildQuotedAtom(Pos, to_lower(ToString(warningPragma.GetAction()))))));
- }
+ }
- if (ctx.ResultSizeLimit > 0) {
+ if (ctx.ResultSizeLimit > 0) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", resultSink,
- BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit)))));
- }
-
+ BuildQuotedAtom(Pos, "SizeLimit"), BuildQuotedAtom(Pos, ToString(ctx.ResultSizeLimit)))));
+ }
+
if (!ctx.PragmaPullUpFlatMapOverJoin) {
Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
BuildQuotedAtom(Pos, "DisablePullUpFlatMapOverJoin"))));
}
-
- if (ctx.DiscoveryMode) {
- Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
- BuildQuotedAtom(Pos, "DiscoveryMode"))));
- }
+
+ if (ctx.DiscoveryMode) {
+ Add(Y("let", "world", Y(TString(ConfigureName), "world", configSource,
+ BuildQuotedAtom(Pos, "DiscoveryMode"))));
+ }
if (ctx.DqEngineEnable) {
TString mode = "auto";
@@ -1765,57 +1765,57 @@ public:
return !hasError;
}
- TPtr DoClone() const final {
- return {};
- }
-private:
+ TPtr DoClone() const final {
+ return {};
+ }
+private:
TVector<TNodePtr> Blocks;
const bool TopLevel;
TScopedStatePtr Scoped;
-};
+};
TNodePtr BuildQuery(TPosition pos, const TVector<TNodePtr>& blocks, bool topLevel, TScopedStatePtr scoped) {
return new TYqlProgramNode(pos, blocks, topLevel, scoped);
-}
-
+}
+
class TPragmaNode final: public INode {
-public:
+public:
TPragmaNode(TPosition pos, const TString& prefix, const TString& name, const TVector<TDeferredAtom>& values, bool valueDefault)
: INode(pos)
- , Prefix(prefix)
- , Name(name)
- , Values(values)
- , ValueDefault(valueDefault)
+ , Prefix(prefix)
+ , Name(name)
+ , Values(values)
+ , ValueDefault(valueDefault)
{
FakeSource = BuildFakeSource(pos);
}
-
- bool DoInit(TContext& ctx, ISource* src) override {
+
+ bool DoInit(TContext& ctx, ISource* src) override {
Y_UNUSED(src);
- TString serviceName;
- TString cluster;
+ TString serviceName;
+ TString cluster;
if (std::find(Providers.cbegin(), Providers.cend(), Prefix) != Providers.cend()) {
- cluster = "$all";
- serviceName = Prefix;
+ cluster = "$all";
+ serviceName = Prefix;
} else {
serviceName = *ctx.GetClusterProvider(Prefix, cluster);
}
- auto datasource = Y("DataSource", BuildQuotedAtom(Pos, serviceName));
- if (Prefix != ConfigProviderName) {
- datasource = L(datasource, BuildQuotedAtom(Pos, cluster));
- }
-
+ auto datasource = Y("DataSource", BuildQuotedAtom(Pos, serviceName));
+ if (Prefix != ConfigProviderName) {
+ datasource = L(datasource, BuildQuotedAtom(Pos, cluster));
+ }
+
Node = Y();
Node = L(Node, AstNode(TString(ConfigureName)));
Node = L(Node, AstNode(TString(TStringBuf("world"))));
Node = L(Node, datasource);
-
+
if (Name == TStringBuf("flags")) {
- for (ui32 i = 0; i < Values.size(); ++i) {
+ for (ui32 i = 0; i < Values.size(); ++i) {
Node = L(Node, Values[i].Build());
- }
- }
+ }
+ }
else if (Name == TStringBuf("AddFileByUrl") || Name == TStringBuf("AddFolderByUrl") || Name == TStringBuf("ImportUdfs") || Name == TStringBuf("SetPackageVersion")) {
Node = L(Node, BuildQuotedAtom(Pos, Name));
for (ui32 i = 0; i < Values.size(); ++i) {
@@ -1825,15 +1825,15 @@ public:
else if (Name == TStringBuf("auth")) {
Node = L(Node, BuildQuotedAtom(Pos, "Auth"));
Node = L(Node, Values.empty() ? BuildQuotedAtom(Pos, TString()) : Values.front().Build());
- }
- else {
+ }
+ else {
Node = L(Node, BuildQuotedAtom(Pos, "Attr"));
Node = L(Node, BuildQuotedAtom(Pos, Name));
- if (!ValueDefault) {
+ if (!ValueDefault) {
Node = L(Node, Values.empty() ? BuildQuotedAtom(Pos, TString()) : Values.front().Build());
- }
- }
-
+ }
+ }
+
if (!Node->Init(ctx, FakeSource.Get())) {
return false;
}
@@ -1848,18 +1848,18 @@ public:
TPtr DoClone() const final {
return {};
}
-
+
private:
- TString Prefix;
- TString Name;
+ TString Prefix;
+ TString Name;
TVector<TDeferredAtom> Values;
- bool ValueDefault;
+ bool ValueDefault;
TNodePtr Node;
TSourcePtr FakeSource;
};
TNodePtr BuildPragma(TPosition pos, const TString& prefix, const TString& name, const TVector<TDeferredAtom>& values, bool valueDefault) {
- return new TPragmaNode(pos, prefix, name, values, valueDefault);
+ return new TPragmaNode(pos, prefix, name, values, valueDefault);
}
class TSqlLambda final: public TAstListNode {
diff --git a/ydb/library/yql/sql/v1/select.cpp b/ydb/library/yql/sql/v1/select.cpp
index 2841f05a5b..3015110239 100644
--- a/ydb/library/yql/sql/v1/select.cpp
+++ b/ydb/library/yql/sql/v1/select.cpp
@@ -19,7 +19,7 @@ public:
, Source(std::move(source))
, Alias(alias)
, InSubquery(inSubquery)
- , EnsureTupleSize(ensureTupleSize)
+ , EnsureTupleSize(ensureTupleSize)
, Scoped(scoped)
{
YQL_ENSURE(!Alias.empty());
@@ -48,9 +48,9 @@ public:
if (!source) {
return false;
}
- if (EnsureTupleSize != -1) {
- source = Y("EnsureTupleSize", source, Q(ToString(EnsureTupleSize)));
- }
+ if (EnsureTupleSize != -1) {
+ source = Y("EnsureTupleSize", source, Q(ToString(EnsureTupleSize)));
+ }
Node = Y("let", Alias, Y("block", Q(L(tables, Y("return", Q(Y("world", source)))))));
IsUsed = true;
@@ -83,7 +83,7 @@ protected:
TNodePtr Node;
const TString Alias;
const bool InSubquery;
- const int EnsureTupleSize;
+ const int EnsureTupleSize;
bool IsUsed = false;
TScopedStatePtr Scoped;
};
@@ -231,7 +231,7 @@ public:
}
TNodePtr BuildFilter(TContext& ctx, const TString& label) override {
- Y_UNUSED(ctx);
+ Y_UNUSED(ctx);
Y_UNUSED(label);
return nullptr;
}
@@ -311,7 +311,7 @@ public:
members = L(members, BuildQuotedAtom(Pos, column));
}
- return Y(ctx.UseUnordered(*this) ? "OrderedMap" : "Map", nodeAst, BuildLambda(Pos, Y("row"), Y("SelectMembers", "row", Q(members))));
+ return Y(ctx.UseUnordered(*this) ? "OrderedMap" : "Map", nodeAst, BuildLambda(Pos, Y("row"), Y("SelectMembers", "row", Q(members))));
}
}
@@ -371,11 +371,11 @@ protected:
return Source->IsStream();
}
- EOrderKind GetOrderKind() const override {
- Y_VERIFY_DEBUG(Source);
- return Source->GetOrderKind();
- }
-
+ EOrderKind GetOrderKind() const override {
+ Y_VERIFY_DEBUG(Source);
+ return Source->GetOrderKind();
+ }
+
TWriteSettings GetWriteSettings() const override {
Y_VERIFY_DEBUG(Source);
return Source->GetWriteSettings();
@@ -442,114 +442,114 @@ protected:
TColumns Columns;
};
-class TMuxSource: public ISource {
-public:
- TMuxSource(TPosition pos, TVector<TSourcePtr>&& sources)
- : ISource(pos)
- , Sources(std::move(sources))
- {
- YQL_ENSURE(Sources.size() > 1);
- }
-
- void AllColumns() final {
- for (auto& source: Sources) {
- source->AllColumns();
- }
- }
-
- const TColumns* GetColumns() const final {
- // Columns are equal in all sources. Return from the first one
- return Sources.front()->GetColumns();
- }
-
- void GetInputTables(TTableList& tableList) const final {
- for (auto& source: Sources) {
- source->GetInputTables(tableList);
- }
- ISource::GetInputTables(tableList);
- }
-
- bool IsStream() const final {
- return AnyOf(Sources, [] (const TSourcePtr& s) { return s->IsStream(); });
- }
-
- bool DoInit(TContext& ctx, ISource* src) final {
- for (auto& source: Sources) {
- if (AsInner) {
- source->UseAsInner();
- }
-
- if (src) {
- src->AddDependentSource(source.Get());
- }
- if (!source->Init(ctx, src)) {
- return false;
- }
- if (!source->InitFilters(ctx)) {
- return false;
- }
- }
- return true;
- }
-
- TMaybe<bool> AddColumn(TContext& ctx, TColumnNode& column) final {
- for (auto& source: Sources) {
- if (!source->AddColumn(ctx, column)) {
- return {};
- }
- }
- return true;
- }
-
- TNodePtr Build(TContext& ctx) final {
- TNodePtr block;
- auto muxArgs = Y();
- for (size_t i = 0; i < Sources.size(); ++i) {
- auto& source = Sources[i];
- auto input = source->Build(ctx);
- auto ref = ctx.MakeName("src");
- muxArgs->Add(ref);
- if (block) {
- block = L(block, Y("let", ref, input));
- } else {
- block = Y(Y("let", ref, input));
- }
+class TMuxSource: public ISource {
+public:
+ TMuxSource(TPosition pos, TVector<TSourcePtr>&& sources)
+ : ISource(pos)
+ , Sources(std::move(sources))
+ {
+ YQL_ENSURE(Sources.size() > 1);
+ }
+
+ void AllColumns() final {
+ for (auto& source: Sources) {
+ source->AllColumns();
+ }
+ }
+
+ const TColumns* GetColumns() const final {
+ // Columns are equal in all sources. Return from the first one
+ return Sources.front()->GetColumns();
+ }
+
+ void GetInputTables(TTableList& tableList) const final {
+ for (auto& source: Sources) {
+ source->GetInputTables(tableList);
+ }
+ ISource::GetInputTables(tableList);
+ }
+
+ bool IsStream() const final {
+ return AnyOf(Sources, [] (const TSourcePtr& s) { return s->IsStream(); });
+ }
+
+ bool DoInit(TContext& ctx, ISource* src) final {
+ for (auto& source: Sources) {
+ if (AsInner) {
+ source->UseAsInner();
+ }
+
+ if (src) {
+ src->AddDependentSource(source.Get());
+ }
+ if (!source->Init(ctx, src)) {
+ return false;
+ }
+ if (!source->InitFilters(ctx)) {
+ return false;
+ }
+ }
+ return true;
+ }
+
+ TMaybe<bool> AddColumn(TContext& ctx, TColumnNode& column) final {
+ for (auto& source: Sources) {
+ if (!source->AddColumn(ctx, column)) {
+ return {};
+ }
+ }
+ return true;
+ }
+
+ TNodePtr Build(TContext& ctx) final {
+ TNodePtr block;
+ auto muxArgs = Y();
+ for (size_t i = 0; i < Sources.size(); ++i) {
+ auto& source = Sources[i];
+ auto input = source->Build(ctx);
+ auto ref = ctx.MakeName("src");
+ muxArgs->Add(ref);
+ if (block) {
+ block = L(block, Y("let", ref, input));
+ } else {
+ block = Y(Y("let", ref, input));
+ }
auto filter = source->BuildFilter(ctx, ref);
- if (filter) {
- block = L(block, Y("let", ref, filter));
- }
- if (ctx.EnableSystemColumns) {
- block = L(block, Y("let", ref, Y("RemoveSystemMembers", ref)));
- }
- }
- return GroundWithExpr(block, Y("Mux", Q(muxArgs)));
- }
-
- bool AddFilter(TContext& ctx, TNodePtr filter) final {
- Y_UNUSED(filter);
- ctx.Error() << "Filter is not allowed for multiple sources";
- return false;
- }
-
- TPtr DoClone() const final {
- return new TMuxSource(Pos, CloneContainer(Sources));
- }
-
-protected:
- TVector<TSourcePtr> Sources;
-};
-
-TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources) {
- return new TMuxSource(pos, std::move(sources));
-}
-
+ if (filter) {
+ block = L(block, Y("let", ref, filter));
+ }
+ if (ctx.EnableSystemColumns) {
+ block = L(block, Y("let", ref, Y("RemoveSystemMembers", ref)));
+ }
+ }
+ return GroundWithExpr(block, Y("Mux", Q(muxArgs)));
+ }
+
+ bool AddFilter(TContext& ctx, TNodePtr filter) final {
+ Y_UNUSED(filter);
+ ctx.Error() << "Filter is not allowed for multiple sources";
+ return false;
+ }
+
+ TPtr DoClone() const final {
+ return new TMuxSource(Pos, CloneContainer(Sources));
+ }
+
+protected:
+ TVector<TSourcePtr> Sources;
+};
+
+TSourcePtr BuildMuxSource(TPosition pos, TVector<TSourcePtr>&& sources) {
+ return new TMuxSource(pos, std::move(sources));
+}
+
class TSubqueryRefNode: public IRealSource {
public:
- TSubqueryRefNode(const TNodePtr& subquery, const TString& alias, int tupleIndex)
+ TSubqueryRefNode(const TNodePtr& subquery, const TString& alias, int tupleIndex)
: IRealSource(subquery->GetPos())
, Subquery(subquery)
, Alias(alias)
- , TupleIndex(tupleIndex)
+ , TupleIndex(tupleIndex)
{
YQL_ENSURE(subquery->GetSource());
}
@@ -566,9 +566,9 @@ public:
}
Columns = *Subquery->GetSource()->GetColumns();
Node = BuildAtom(Pos, Alias, TNodeFlags::Default);
- if (TupleIndex != -1) {
- Node = Y("Nth", Node, Q(ToString(TupleIndex)));
- }
+ if (TupleIndex != -1) {
+ Node = Y("Nth", Node, Q(ToString(TupleIndex)));
+ }
if (!Node->Init(ctx, src)) {
return false;
}
@@ -628,18 +628,18 @@ public:
}
TPtr DoClone() const final {
- return new TSubqueryRefNode(Subquery, Alias, TupleIndex);
+ return new TSubqueryRefNode(Subquery, Alias, TupleIndex);
}
protected:
TNodePtr Subquery;
const TString Alias;
- const int TupleIndex;
+ const int TupleIndex;
TNodePtr Node;
};
-TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleIndex) {
- return new TSubqueryRefNode(std::move(subquery), alias, tupleIndex);
+TNodePtr BuildSubqueryRef(TNodePtr subquery, const TString& alias, int tupleIndex) {
+ return new TSubqueryRefNode(std::move(subquery), alias, tupleIndex);
}
class TTableSource: public IRealSource {
@@ -710,9 +710,9 @@ public:
auto sampleSettings = Q(Y(Q(modeName), Y("EvaluateAtom", Y("ToString", samplingRate)), Y("EvaluateAtom", Y("ToString", samplingSeed))));
auto sampleOption = Q(Y(Q("sample"), sampleSettings));
if (Table.Options) {
- if (!Table.Options->Init(ctx, this)) {
- return false;
- }
+ if (!Table.Options->Init(ctx, this)) {
+ return false;
+ }
Table.Options = L(Table.Options, sampleOption);
} else {
Table.Options = Y(sampleOption);
@@ -898,30 +898,30 @@ TSourcePtr BuildInnerSource(TPosition pos, TNodePtr node, const TString& service
return new TInnerSource(pos, node, service, cluster, label);
}
-static bool IsComparableExpression(TContext& ctx, const TNodePtr& expr, bool assume, const char* sqlConstruction) {
- if (assume && !expr->GetColumnName()) {
- ctx.Error(expr->GetPos()) << "Only column names can be used in " << sqlConstruction;
- return false;
- }
-
- if (expr->IsConstant()) {
- ctx.Error(expr->GetPos()) << "Unable to " << sqlConstruction << " constant expression";
- return false;
- }
- if (expr->IsAggregated() && !expr->HasState(ENodeState::AggregationKey)) {
- ctx.Error(expr->GetPos()) << "Unable to " << sqlConstruction << " aggregated values";
- return false;
- }
+static bool IsComparableExpression(TContext& ctx, const TNodePtr& expr, bool assume, const char* sqlConstruction) {
+ if (assume && !expr->GetColumnName()) {
+ ctx.Error(expr->GetPos()) << "Only column names can be used in " << sqlConstruction;
+ return false;
+ }
+
+ if (expr->IsConstant()) {
+ ctx.Error(expr->GetPos()) << "Unable to " << sqlConstruction << " constant expression";
+ return false;
+ }
+ if (expr->IsAggregated() && !expr->HasState(ENodeState::AggregationKey)) {
+ ctx.Error(expr->GetPos()) << "Unable to " << sqlConstruction << " aggregated values";
+ return false;
+ }
if (expr->GetColumnName()) {
- return true;
- }
- if (expr->GetOpName().empty()) {
- ctx.Error(expr->GetPos()) << "You should use in " << sqlConstruction << " column name, qualified field, callable function or expression";
- return false;
- }
- return true;
-}
-
+ return true;
+ }
+ if (expr->GetOpName().empty()) {
+ ctx.Error(expr->GetPos()) << "You should use in " << sqlConstruction << " column name, qualified field, callable function or expression";
+ return false;
+ }
+ return true;
+}
+
/// \todo move to reduce.cpp? or mapreduce.cpp?
class TReduceSource: public IRealSource {
public:
@@ -933,7 +933,7 @@ public:
TVector<TNodePtr>&& args,
TNodePtr udf,
TNodePtr having,
- const TWriteSettings& settings,
+ const TWriteSettings& settings,
const TVector<TSortSpecificationPtr>& assumeOrderBy,
bool listCall)
: IRealSource(pos)
@@ -945,7 +945,7 @@ public:
, Udf(udf)
, Having(having)
, Settings(settings)
- , AssumeOrderBy(assumeOrderBy)
+ , AssumeOrderBy(assumeOrderBy)
, ListCall(listCall)
{
YQL_ENSURE(!Keys.empty());
@@ -1008,20 +1008,20 @@ public:
} else {
Columns.Add(&Udf->GetLabel(), false);
}
-
- const auto label = GetLabel();
- for (const auto& sortSpec: AssumeOrderBy) {
- auto& expr = sortSpec->OrderExpr;
- SetLabel(Source->GetLabel());
- if (!expr->Init(ctx, this)) {
- return false;
- }
- if (!IsComparableExpression(ctx, expr, true, "ASSUME ORDER BY")) {
- return false;
- }
- }
- SetLabel(label);
-
+
+ const auto label = GetLabel();
+ for (const auto& sortSpec: AssumeOrderBy) {
+ auto& expr = sortSpec->OrderExpr;
+ SetLabel(Source->GetLabel());
+ if (!expr->Init(ctx, this)) {
+ return false;
+ }
+ if (!IsComparableExpression(ctx, expr, true, "ASSUME ORDER BY")) {
+ return false;
+ }
+ }
+ SetLabel(label);
+
return true;
}
@@ -1031,17 +1031,17 @@ public:
return nullptr;
}
- auto keysTuple = Y();
- if (Keys.size() == 1) {
- keysTuple = Y("Member", "row", BuildQuotedAtom(Pos, *Keys.back()->GetColumnName()));
- }
- else {
+ auto keysTuple = Y();
+ if (Keys.size() == 1) {
+ keysTuple = Y("Member", "row", BuildQuotedAtom(Pos, *Keys.back()->GetColumnName()));
+ }
+ else {
for (const auto& key: Keys) {
keysTuple = L(keysTuple, Y("Member", "row", BuildQuotedAtom(Pos, *key->GetColumnName())));
}
keysTuple = Q(keysTuple);
}
- auto extractKey = Y("SqlExtractKey", "row", BuildLambda(Pos, Y("row"), keysTuple));
+ auto extractKey = Y("SqlExtractKey", "row", BuildLambda(Pos, Y("row"), keysTuple));
auto extractKeyLambda = BuildLambda(Pos, Y("row"), extractKey);
TNodePtr processPartitions;
@@ -1079,9 +1079,9 @@ public:
TNodePtr sortDirection;
TNodePtr sortKeySelector;
FillSortParts(OrderBy, sortDirection, sortKeySelector);
- if (!OrderBy.empty()) {
- sortKeySelector = BuildLambda(Pos, Y("row"), Y("SqlExtractKey", "row", sortKeySelector));
- }
+ if (!OrderBy.empty()) {
+ sortKeySelector = BuildLambda(Pos, Y("row"), Y("SqlExtractKey", "row", sortKeySelector));
+ }
auto partitionByKey = Y(!ListCall && Mode == ReduceMode::ByAll ? "PartitionByKey" : "PartitionsByKeys", "core", extractKeyLambda,
sortDirection, sortKeySelector, BuildLambda(Pos, Y("partitionStream"), processPartitions));
@@ -1096,10 +1096,10 @@ public:
block = L(block, Y("let", "core", "inputRowsList"));
}
- if (ctx.EnableSystemColumns) {
- block = L(block, Y("let", "core", Y("RemoveSystemMembers", "core")));
- }
- block = L(block, Y("let", "core", Y("AutoDemuxList", partitionByKey)));
+ if (ctx.EnableSystemColumns) {
+ block = L(block, Y("let", "core", Y("RemoveSystemMembers", "core")));
+ }
+ block = L(block, Y("let", "core", Y("AutoDemuxList", partitionByKey)));
if (Having) {
block = L(block, Y("let", "core",
Y("Filter", "core", BuildLambda(Pos, Y("row"), Y("Coalesce", Having, Y("Bool", Q("false")))))
@@ -1108,19 +1108,19 @@ public:
return Y("block", Q(L(block, Y("return", "core"))));
}
- TNodePtr BuildSort(TContext& ctx, const TString& label) override {
- Y_UNUSED(ctx);
- if (AssumeOrderBy.empty()) {
- return nullptr;
- }
-
+ TNodePtr BuildSort(TContext& ctx, const TString& label) override {
+ Y_UNUSED(ctx);
+ if (AssumeOrderBy.empty()) {
+ return nullptr;
+ }
+
return Y("let", label, BuildSortSpec(AssumeOrderBy, label, false, true));
- }
-
- EOrderKind GetOrderKind() const override {
- return AssumeOrderBy.empty() ? EOrderKind::None : EOrderKind::Assume;
- }
-
+ }
+
+ EOrderKind GetOrderKind() const override {
+ return AssumeOrderBy.empty() ? EOrderKind::None : EOrderKind::Assume;
+ }
+
TWriteSettings GetWriteSettings() const final {
return Settings;
}
@@ -1139,7 +1139,7 @@ private:
TNodePtr Udf;
TNodePtr Having;
const TWriteSettings Settings;
- TVector<TSortSpecificationPtr> AssumeOrderBy;
+ TVector<TSortSpecificationPtr> AssumeOrderBy;
const bool ListCall;
};
@@ -1151,7 +1151,7 @@ TSourcePtr BuildReduce(TPosition pos,
TVector<TNodePtr>&& args,
TNodePtr udf,
TNodePtr having,
- const TWriteSettings& settings,
+ const TWriteSettings& settings,
const TVector<TSortSpecificationPtr>& assumeOrderBy,
bool listCall) {
return new TReduceSource(pos, mode, std::move(source), std::move(orderBy), std::move(keys),
@@ -1325,10 +1325,10 @@ public:
return Subselects.front()->BuildSort(ctx, label);
}
- EOrderKind GetOrderKind() const override {
- return Subselects.front()->GetOrderKind();
- }
-
+ EOrderKind GetOrderKind() const override {
+ return Subselects.front()->GetOrderKind();
+ }
+
const TColumns* GetColumns() const override{
return Subselects.front()->GetColumns();
}
@@ -1396,7 +1396,7 @@ public:
const TVector<TNodePtr>& groupByExpr,
const TVector<TNodePtr>& groupBy,
bool compactGroupBy,
- bool assumeSorted,
+ bool assumeSorted,
const TVector<TSortSpecificationPtr>& orderBy,
TNodePtr having,
TWinSpecs& winSpecs,
@@ -1425,12 +1425,12 @@ public:
{
}
- void AllColumns() override {
- if (!OrderByInit) {
- Columns.SetAll();
- }
- }
-
+ void AllColumns() override {
+ if (!OrderByInit) {
+ Columns.SetAll();
+ }
+ }
+
void GetInputTables(TTableList& tableList) const override {
Source->GetInputTables(tableList);
ISource::GetInputTables(tableList);
@@ -1494,7 +1494,7 @@ public:
}
DistinctAggrExpr.insert(DistinctAggrExpr.end(), distinctAggrsInGroupBy.begin(), distinctAggrsInGroupBy.end());
- if (!expr->Init(ctx, src) || !IsComparableExpression(ctx, expr, false, "GROUP BY")) {
+ if (!expr->Init(ctx, src) || !IsComparableExpression(ctx, expr, false, "GROUP BY")) {
hasError = true;
}
}
@@ -1537,8 +1537,8 @@ public:
src->AddWindowSpecs(WinSpecs);
const bool isJoin = Source->GetJoin();
- if (!InitSelect(ctx, src, isJoin, hasError)) {
- return false;
+ if (!InitSelect(ctx, src, isJoin, hasError)) {
+ return false;
}
src->FinishColumns();
@@ -1619,13 +1619,13 @@ public:
return nullptr;
}
- bool ordered = ctx.UseUnordered(*this);
+ bool ordered = ctx.UseUnordered(*this);
auto block(Y(Y("let", "core", input)));
if (PreFlattenMap) {
block = L(block, Y("let", "core", Y(ordered ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), PreFlattenMap))));
}
if (Flatten) {
- block = L(block, Y("let", "core", Y(ordered ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), Flatten, "res"))));
+ block = L(block, Y("let", "core", Y(ordered ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), Flatten, "res"))));
}
if (PreaggregatedMap) {
block = L(block, Y("let", "core", PreaggregatedMap));
@@ -1641,7 +1641,7 @@ public:
}
if (Aggregate) {
block = L(block, Y("let", "core", Aggregate));
- ordered = false;
+ ordered = false;
}
if (PrewindowMap) {
block = L(block, Y("let", "core", PrewindowMap));
@@ -1668,51 +1668,51 @@ public:
return Y("let", label, BuildSortSpec(OrderBy, label, false, AssumeSorted));
}
- TNodePtr BuildCleanupColumns(TContext& ctx, const TString& label) override {
- TNodePtr cleanup;
- if (ctx.EnableSystemColumns && ctx.Settings.Mode != NSQLTranslation::ESqlMode::LIMITED_VIEW) {
- if (Columns.All) {
- cleanup = Y("let", label, Y("RemoveSystemMembers", label));
- } else if (!Columns.List.empty()) {
- const bool isJoin = Source->GetJoin();
- if (!isJoin && Columns.QualifiedAll) {
- if (ctx.SimpleColumns) {
- cleanup = Y("let", label, Y("RemoveSystemMembers", label));
- } else {
- TNodePtr members;
- for (auto& term: Terms) {
- if (term->IsAsterisk()) {
- auto sourceName = term->GetSourceName();
- YQL_ENSURE(*sourceName && !sourceName->empty());
- auto prefix = *sourceName + "._yql_";
- members = members ? L(members, Q(prefix)) : Y(Q(prefix));
- }
- }
- if (members) {
- cleanup = Y("let", label, Y("RemovePrefixMembers", label, Q(members)));
- }
- }
- }
- }
- }
- return cleanup;
- }
-
+ TNodePtr BuildCleanupColumns(TContext& ctx, const TString& label) override {
+ TNodePtr cleanup;
+ if (ctx.EnableSystemColumns && ctx.Settings.Mode != NSQLTranslation::ESqlMode::LIMITED_VIEW) {
+ if (Columns.All) {
+ cleanup = Y("let", label, Y("RemoveSystemMembers", label));
+ } else if (!Columns.List.empty()) {
+ const bool isJoin = Source->GetJoin();
+ if (!isJoin && Columns.QualifiedAll) {
+ if (ctx.SimpleColumns) {
+ cleanup = Y("let", label, Y("RemoveSystemMembers", label));
+ } else {
+ TNodePtr members;
+ for (auto& term: Terms) {
+ if (term->IsAsterisk()) {
+ auto sourceName = term->GetSourceName();
+ YQL_ENSURE(*sourceName && !sourceName->empty());
+ auto prefix = *sourceName + "._yql_";
+ members = members ? L(members, Q(prefix)) : Y(Q(prefix));
+ }
+ }
+ if (members) {
+ cleanup = Y("let", label, Y("RemovePrefixMembers", label, Q(members)));
+ }
+ }
+ }
+ }
+ }
+ return cleanup;
+ }
+
bool IsSelect() const override {
- return true;
+ return true;
}
bool IsStream() const override {
return Source->IsStream();
}
- EOrderKind GetOrderKind() const override {
- if (OrderBy.empty()) {
- return EOrderKind::None;
- }
- return AssumeSorted ? EOrderKind::Assume : EOrderKind::Sort;
- }
-
+ EOrderKind GetOrderKind() const override {
+ if (OrderBy.empty()) {
+ return EOrderKind::None;
+ }
+ return AssumeSorted ? EOrderKind::Assume : EOrderKind::Sort;
+ }
+
TWriteSettings GetWriteSettings() const override {
return Settings;
}
@@ -1755,7 +1755,7 @@ public:
for (auto cur: WinSpecs) {
newSpecs.emplace(cur.first, cur.second->Clone());
}
- return new TSelectCore(Pos, Source->CloneSource(), CloneContainer(GroupByExpr),
+ return new TSelectCore(Pos, Source->CloneSource(), CloneContainer(GroupByExpr),
CloneContainer(GroupBy), CompactGroupBy, AssumeSorted, CloneContainer(OrderBy),
SafeClone(Having), newSpecs, SafeClone(HoppingWindowSpec),
CloneContainer(Terms), Distinct, Without, SelectStream, Settings);
@@ -1898,7 +1898,7 @@ private:
continue;
}
OrderByInit = false;
- if (!IsComparableExpression(ctx, expr, AssumeSorted, AssumeSorted ? "ASSUME ORDER BY" : "ORDER BY")) {
+ if (!IsComparableExpression(ctx, expr, AssumeSorted, AssumeSorted ? "ASSUME ORDER BY" : "ORDER BY")) {
hasError = true;
continue;
}
@@ -2075,7 +2075,7 @@ private:
TVector<TNodePtr> GroupByExpr;
TVector<TNodePtr> DistinctAggrExpr;
TVector<TNodePtr> GroupBy;
- bool AssumeSorted = false;
+ bool AssumeSorted = false;
bool CompactGroupBy = false;
TVector<TSortSpecificationPtr> OrderBy;
TNodePtr Having;
@@ -2096,51 +2096,51 @@ private:
const TWriteSettings Settings;
};
-class TProcessSource: public IRealSource {
-public:
- TProcessSource(
- TPosition pos,
- TSourcePtr source,
- TNodePtr with,
+class TProcessSource: public IRealSource {
+public:
+ TProcessSource(
+ TPosition pos,
+ TSourcePtr source,
+ TNodePtr with,
bool withExtFunction,
TVector<TNodePtr>&& terms,
- bool listCall,
+ bool listCall,
bool processStream,
- const TWriteSettings& settings,
- const TVector<TSortSpecificationPtr>& assumeOrderBy
- )
- : IRealSource(pos)
- , Source(std::move(source))
- , With(with)
+ const TWriteSettings& settings,
+ const TVector<TSortSpecificationPtr>& assumeOrderBy
+ )
+ : IRealSource(pos)
+ , Source(std::move(source))
+ , With(with)
, WithExtFunction(withExtFunction)
, Terms(std::move(terms))
- , ListCall(listCall)
+ , ListCall(listCall)
, ProcessStream(processStream)
, Settings(settings)
- , AssumeOrderBy(assumeOrderBy)
- {
- }
-
+ , AssumeOrderBy(assumeOrderBy)
+ {
+ }
+
void GetInputTables(TTableList& tableList) const override {
Source->GetInputTables(tableList);
ISource::GetInputTables(tableList);
- }
-
- bool DoInit(TContext& ctx, ISource* initSrc) override {
- if (AsInner) {
- Source->UseAsInner();
- }
-
- if (!Source->Init(ctx, initSrc)) {
- return false;
- }
-
+ }
+
+ bool DoInit(TContext& ctx, ISource* initSrc) override {
+ if (AsInner) {
+ Source->UseAsInner();
+ }
+
+ if (!Source->Init(ctx, initSrc)) {
+ return false;
+ }
+
if (ProcessStream && !Source->IsStream()) {
ctx.Error(Pos) << "PROCESS STREAM is unsupported for non-streaming sources";
return false;
}
- auto src = Source.Get();
+ auto src = Source.Get();
if (!With) {
src->AllColumns();
Columns.SetAll();
@@ -2148,32 +2148,32 @@ public:
return true;
}
- /// grouped expressions are available in filters
- if (!Source->InitFilters(ctx)) {
- return false;
- }
-
+ /// grouped expressions are available in filters
+ if (!Source->InitFilters(ctx)) {
+ return false;
+ }
+
TSourcePtr fakeSource = nullptr;
if (ListCall && !WithExtFunction) {
- fakeSource = BuildFakeSource(src->GetPos());
- src->AllColumns();
- }
-
+ fakeSource = BuildFakeSource(src->GetPos());
+ src->AllColumns();
+ }
+
auto processSource = fakeSource != nullptr ? fakeSource.Get() : src;
- Y_VERIFY_DEBUG(processSource != nullptr);
- if (!With->Init(ctx, processSource)) {
- return false;
- }
- if (With->GetLabel().empty()) {
- Columns.SetAll();
- } else {
- if (ListCall) {
+ Y_VERIFY_DEBUG(processSource != nullptr);
+ if (!With->Init(ctx, processSource)) {
+ return false;
+ }
+ if (With->GetLabel().empty()) {
+ Columns.SetAll();
+ } else {
+ if (ListCall) {
ctx.Error(With->GetPos()) << "Label is not allowed to use with TableRows()";
- return false;
- }
+ return false;
+ }
Columns.Add(&With->GetLabel(), false);
- }
-
+ }
+
bool hasError = false;
TNodePtr produce;
@@ -2183,91 +2183,91 @@ public:
TString processCall = (ListCall ? "SqlProcess" : "Apply");
produce = Y(processCall, With);
}
- TMaybe<ui32> listPosIndex;
- ui32 termIndex = 0;
- for (auto& term: Terms) {
- if (!term->GetLabel().empty()) {
- ctx.Error(term->GetPos()) << "Labels are not allowed for PROCESS terms";
- hasError = true;
- continue;
- }
-
- if (!term->Init(ctx, processSource)) {
- hasError = true;
- continue;
- }
-
- if (ListCall) {
- if (auto atom = dynamic_cast<TTableRows*>(term.Get())) {
- listPosIndex = termIndex;
- }
- }
- ++termIndex;
-
- produce = L(produce, term);
- }
-
+ TMaybe<ui32> listPosIndex;
+ ui32 termIndex = 0;
+ for (auto& term: Terms) {
+ if (!term->GetLabel().empty()) {
+ ctx.Error(term->GetPos()) << "Labels are not allowed for PROCESS terms";
+ hasError = true;
+ continue;
+ }
+
+ if (!term->Init(ctx, processSource)) {
+ hasError = true;
+ continue;
+ }
+
+ if (ListCall) {
+ if (auto atom = dynamic_cast<TTableRows*>(term.Get())) {
+ listPosIndex = termIndex;
+ }
+ }
+ ++termIndex;
+
+ produce = L(produce, term);
+ }
+
if (hasError) {
return false;
}
if (ListCall && !WithExtFunction) {
YQL_ENSURE(listPosIndex.Defined());
- produce = L(produce, Q(ToString(*listPosIndex)));
- }
-
- if (!produce->Init(ctx, src)) {
- hasError = true;
- }
-
+ produce = L(produce, Q(ToString(*listPosIndex)));
+ }
+
+ if (!produce->Init(ctx, src)) {
+ hasError = true;
+ }
+
if (!(WithExtFunction && Terms.empty())) {
TVector<TNodePtr>(1, produce).swap(Terms);
}
-
- src->FinishColumns();
-
- const auto label = GetLabel();
- for (const auto& sortSpec: AssumeOrderBy) {
- auto& expr = sortSpec->OrderExpr;
- SetLabel(Source->GetLabel());
- if (!expr->Init(ctx, this)) {
- hasError = true;
- continue;
- }
- if (!IsComparableExpression(ctx, expr, true, "ASSUME ORDER BY")) {
- hasError = true;
- continue;
- }
- }
- SetLabel(label);
-
- return !hasError;
- }
-
- TNodePtr Build(TContext& ctx) override {
- auto input = Source->Build(ctx);
- if (!input) {
- return nullptr;
- }
-
+
+ src->FinishColumns();
+
+ const auto label = GetLabel();
+ for (const auto& sortSpec: AssumeOrderBy) {
+ auto& expr = sortSpec->OrderExpr;
+ SetLabel(Source->GetLabel());
+ if (!expr->Init(ctx, this)) {
+ hasError = true;
+ continue;
+ }
+ if (!IsComparableExpression(ctx, expr, true, "ASSUME ORDER BY")) {
+ hasError = true;
+ continue;
+ }
+ }
+ SetLabel(label);
+
+ return !hasError;
+ }
+
+ TNodePtr Build(TContext& ctx) override {
+ auto input = Source->Build(ctx);
+ if (!input) {
+ return nullptr;
+ }
+
if (!With) {
- auto res = input;
- if (ctx.EnableSystemColumns) {
- res = Y("RemoveSystemMembers", res);
- }
-
- return res;
- }
-
- TString inputLabel = ListCall ? "inputRowsList" : "core";
-
- auto block(Y(Y("let", inputLabel, input)));
-
+ auto res = input;
+ if (ctx.EnableSystemColumns) {
+ res = Y("RemoveSystemMembers", res);
+ }
+
+ return res;
+ }
+
+ TString inputLabel = ListCall ? "inputRowsList" : "core";
+
+ auto block(Y(Y("let", inputLabel, input)));
+
auto filter = Source->BuildFilter(ctx, inputLabel);
- if (filter) {
- block = L(block, Y("let", inputLabel, filter));
- }
-
+ if (filter) {
+ block = L(block, Y("let", inputLabel, filter));
+ }
+
if (WithExtFunction) {
auto preTransform = Y("RemoveSystemMembers", inputLabel);
if (Terms.size() > 0) {
@@ -2277,89 +2277,89 @@ public:
block = L(block, Y("let", "transform", With));
block = L(block, Y("let", "core", Y("Apply", "transform", inputLabel)));
} else if (ListCall) {
- block = L(block, Y("let", "core", Terms[0]));
- } else {
- auto terms = BuildColumnsTerms(ctx);
- block = L(block, Y("let", "core", Y(ctx.UseUnordered(*this) ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), terms, "res"))));
- }
+ block = L(block, Y("let", "core", Terms[0]));
+ } else {
+ auto terms = BuildColumnsTerms(ctx);
+ block = L(block, Y("let", "core", Y(ctx.UseUnordered(*this) ? "OrderedFlatMap" : "FlatMap", "core", BuildLambda(Pos, Y("row"), terms, "res"))));
+ }
block = L(block, Y("let", "core", Y("AutoDemuxList", Y("PersistableRepr", "core"))));
- return Y("block", Q(L(block, Y("return", "core"))));
- }
-
- TNodePtr BuildSort(TContext& ctx, const TString& label) override {
- Y_UNUSED(ctx);
- if (AssumeOrderBy.empty()) {
- return nullptr;
- }
-
+ return Y("block", Q(L(block, Y("return", "core"))));
+ }
+
+ TNodePtr BuildSort(TContext& ctx, const TString& label) override {
+ Y_UNUSED(ctx);
+ if (AssumeOrderBy.empty()) {
+ return nullptr;
+ }
+
return Y("let", label, BuildSortSpec(AssumeOrderBy, label, false, true));
- }
-
- EOrderKind GetOrderKind() const override {
- if (!With) {
- return EOrderKind::Passthrough;
- }
- return AssumeOrderBy.empty() ? EOrderKind::None : EOrderKind::Assume;
- }
-
- bool IsSelect() const override {
- return false;
- }
-
+ }
+
+ EOrderKind GetOrderKind() const override {
+ if (!With) {
+ return EOrderKind::Passthrough;
+ }
+ return AssumeOrderBy.empty() ? EOrderKind::None : EOrderKind::Assume;
+ }
+
+ bool IsSelect() const override {
+ return false;
+ }
+
bool IsStream() const override {
return Source->IsStream();
}
TWriteSettings GetWriteSettings() const override {
return Settings;
- }
-
- TNodePtr DoClone() const final {
+ }
+
+ TNodePtr DoClone() const final {
return new TProcessSource(Pos, Source->CloneSource(), SafeClone(With), WithExtFunction,
CloneContainer(Terms), ListCall, ProcessStream, Settings, CloneContainer(AssumeOrderBy));
- }
-
-private:
- TNodePtr BuildColumnsTerms(TContext& ctx) {
+ }
+
+private:
+ TNodePtr BuildColumnsTerms(TContext& ctx) {
Y_UNUSED(ctx);
- TNodePtr terms;
- Y_VERIFY_DEBUG(Terms.size() == 1);
- if (Columns.All) {
- terms = Y(Y("let", "res", Y("ToSequence", Terms.front())));
- } else {
- Y_VERIFY_DEBUG(Columns.List.size() == Terms.size());
+ TNodePtr terms;
+ Y_VERIFY_DEBUG(Terms.size() == 1);
+ if (Columns.All) {
+ terms = Y(Y("let", "res", Y("ToSequence", Terms.front())));
+ } else {
+ Y_VERIFY_DEBUG(Columns.List.size() == Terms.size());
terms = L(Y(), Y("let", "res",
L(Y("AsStructUnordered"), Q(Y(BuildQuotedAtom(Pos, Columns.List.front()), Terms.front())))));
- terms = L(terms, Y("let", "res", Y("Just", "res")));
- }
- return terms;
- }
-
-private:
- TSourcePtr Source;
- TNodePtr With;
+ terms = L(terms, Y("let", "res", Y("Just", "res")));
+ }
+ return terms;
+ }
+
+private:
+ TSourcePtr Source;
+ TNodePtr With;
const bool WithExtFunction;
- TVector<TNodePtr> Terms;
- const bool ListCall;
+ TVector<TNodePtr> Terms;
+ const bool ListCall;
const bool ProcessStream;
const TWriteSettings Settings;
- TVector<TSortSpecificationPtr> AssumeOrderBy;
-};
-
-TSourcePtr BuildProcess(
- TPosition pos,
- TSourcePtr source,
- TNodePtr with,
+ TVector<TSortSpecificationPtr> AssumeOrderBy;
+};
+
+TSourcePtr BuildProcess(
+ TPosition pos,
+ TSourcePtr source,
+ TNodePtr with,
bool withExtFunction,
- TVector<TNodePtr>&& terms,
- bool listCall,
+ TVector<TNodePtr>&& terms,
+ bool listCall,
bool processStream,
- const TWriteSettings& settings,
- const TVector<TSortSpecificationPtr>& assumeOrderBy
-) {
+ const TWriteSettings& settings,
+ const TVector<TSortSpecificationPtr>& assumeOrderBy
+) {
return new TProcessSource(pos, std::move(source), with, withExtFunction, std::move(terms), listCall, processStream, settings, assumeOrderBy);
-}
-
+}
+
class TNestedProxySource: public IProxySource {
public:
TNestedProxySource(TPosition pos, const TVector<TNodePtr>& groupBy, TSourcePtr source)
@@ -2457,7 +2457,7 @@ TSourcePtr DoBuildSelectCore(
const TVector<TNodePtr>& groupByExpr,
const TVector<TNodePtr>& groupBy,
bool compactGroupBy,
- bool assumeSorted,
+ bool assumeSorted,
const TVector<TSortSpecificationPtr>& orderBy,
TNodePtr having,
TWinSpecs&& winSpecs,
@@ -2503,7 +2503,7 @@ TSourcePtr DoBuildSelectCore(
std::swap(terms, termsCopy);
}
totalGroups += contentPtr->size();
- TSelectCore* selectCore = new TSelectCore(pos, std::move(proxySource), CloneContainer(groupByExpr),
+ TSelectCore* selectCore = new TSelectCore(pos, std::move(proxySource), CloneContainer(groupByExpr),
CloneContainer(*contentPtr), compactGroupBy, assumeSorted, orderBy, SafeClone(having), winSpecs,
hoppingWindowSpec, terms, distinct, without, selectStream, settings);
subselects.emplace_back(selectCore);
@@ -2762,11 +2762,11 @@ public:
} else if (sample) {
block = L(block, Y("let", "select", Y("OrderedFlatMap", "select", sample)));
}
-
- if (auto removeNode = Source->BuildCleanupColumns(ctx, label)) {
- block = L(block, removeNode);
- }
-
+
+ if (auto removeNode = Source->BuildCleanupColumns(ctx, label)) {
+ block = L(block, removeNode);
+ }
+
block = L(block, Y("return", label));
return Y("block", Q(block));
}
@@ -2896,9 +2896,9 @@ public:
}
node = L(node, Y("let", "output", output));
if (WriteResult) {
- if (EOrderKind::None == Source->GetOrderKind() && ctx.UseUnordered(*Source)) {
- node = L(node, Y("let", "output", Y("Unordered", "output")));
- }
+ if (EOrderKind::None == Source->GetOrderKind() && ctx.UseUnordered(*Source)) {
+ node = L(node, Y("let", "output", Y("Unordered", "output")));
+ }
auto writeResult(BuildWriteResult(Pos, "output", settings));
if (!writeResult->Init(ctx, src)) {
return false;
diff --git a/ydb/library/yql/sql/v1/sql.cpp b/ydb/library/yql/sql/v1/sql.cpp
index ab5a864788..fc965d6680 100644
--- a/ydb/library/yql/sql/v1/sql.cpp
+++ b/ydb/library/yql/sql/v1/sql.cpp
@@ -752,18 +752,18 @@ TTableHints GetContextHints(TContext& ctx) {
}
static TTableHints GetTableFuncHints(TStringBuf funcName) {
- TCiString func(funcName);
- TTableHints res;
- if (func.StartsWith("range") || func.StartsWith("like") || func.StartsWith("regexp") || func.StartsWith("filter")) {
- res.emplace("ignore_non_existing", TVector<TNodePtr>{});
- } else if (func.StartsWith("each")) {
- res.emplace("ignore_non_existing", TVector<TNodePtr>{});
- res.emplace("warn_non_existing", TVector<TNodePtr>{});
- }
-
- return res;
-}
-
+ TCiString func(funcName);
+ TTableHints res;
+ if (func.StartsWith("range") || func.StartsWith("like") || func.StartsWith("regexp") || func.StartsWith("filter")) {
+ res.emplace("ignore_non_existing", TVector<TNodePtr>{});
+ } else if (func.StartsWith("each")) {
+ res.emplace("ignore_non_existing", TVector<TNodePtr>{});
+ res.emplace("warn_non_existing", TVector<TNodePtr>{});
+ }
+
+ return res;
+}
+
static bool ValidateForCounters(const TString& input) {
for (auto c : input) {
if (!(IsAlnum(c) || c == '_')) {
@@ -4830,24 +4830,24 @@ bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& n
hasError = true;
continue;
}
- if (names.size() > 1) {
- auto ref = ctx.MakeName("tie");
- exprSeq.push_back(nodeExpr->Y("EnsureTupleSize", nodeExpr, nodeExpr->Q(ToString(names.size()))));
- exprSeq.back()->SetLabel(ref);
- for (size_t i = 0; i < names.size(); ++i) {
- TNodePtr nthExpr = nodeExpr->Y("Nth", ref, nodeExpr->Q(ToString(i)));
+ if (names.size() > 1) {
+ auto ref = ctx.MakeName("tie");
+ exprSeq.push_back(nodeExpr->Y("EnsureTupleSize", nodeExpr, nodeExpr->Q(ToString(names.size()))));
+ exprSeq.back()->SetLabel(ref);
+ for (size_t i = 0; i < names.size(); ++i) {
+ TNodePtr nthExpr = nodeExpr->Y("Nth", ref, nodeExpr->Q(ToString(i)));
names[i].Name = PushNamedAtom(names[i].Pos, names[i].Name);
nthExpr->SetLabel(names[i].Name);
localNames.push_back(names[i].Name);
- exprSeq.push_back(nthExpr);
- }
- } else {
+ exprSeq.push_back(nthExpr);
+ }
+ } else {
auto& symbol = names.front();
symbol.Name = PushNamedAtom(symbol.Pos, symbol.Name);
nodeExpr->SetLabel(symbol.Name);
localNames.push_back(symbol.Name);
- exprSeq.push_back(nodeExpr);
- }
+ exprSeq.push_back(nodeExpr);
+ }
break;
}
case TRule_lambda_stmt::kAltLambdaStmt2: {
@@ -4857,7 +4857,7 @@ bool TSqlExpression::SqlLambdaExprBody(TContext& ctx, const TRule_lambda_body& n
break;
}
default:
- Y_FAIL("SampleClause: does not correspond to grammar changes");
+ Y_FAIL("SampleClause: does not correspond to grammar changes");
}
}
@@ -5461,7 +5461,7 @@ TNodePtr TSqlTranslation::NamedNode(const TRule_named_nodes_stmt& rule, TVector<
if (!BindList(rule.GetRule_bind_parameter_list1(), names)) {
return {};
}
-
+
TNodePtr nodeExpr = nullptr;
switch (rule.GetBlock3().Alt_case()) {
case TRule_named_nodes_stmt::TBlock3::kAlt1: {
@@ -6322,7 +6322,7 @@ bool TSqlTranslation::RoleParameters(const TRule_create_user_option& node, TRole
TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteSettings& settings, TPosition& selectPos) {
// PROCESS STREAM? named_single_source (COMMA named_single_source)* (USING using_call_expr (AS an_id)?
// (WITH external_call_settings)?
- // (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?)?
+ // (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?)?
Token(node.GetToken1());
TPosition startPos(Ctx.Pos());
@@ -6337,17 +6337,17 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS
if (!source) {
return nullptr;
}
- if (node.GetBlock4().size()) {
- TVector<TSourcePtr> sources(1, source);
- for (auto& s: node.GetBlock4()) {
+ if (node.GetBlock4().size()) {
+ TVector<TSourcePtr> sources(1, source);
+ for (auto& s: node.GetBlock4()) {
sources.push_back(NamedSingleSource(s.GetRule_named_single_source2(), unorderedSubquery));
- if (!sources.back()) {
- return nullptr;
- }
- }
- auto pos = source->GetPos();
- source = BuildMuxSource(pos, std::move(sources));
- }
+ if (!sources.back()) {
+ return nullptr;
+ }
+ }
+ auto pos = source->GetPos();
+ source = BuildMuxSource(pos, std::move(sources));
+ }
const bool processStream = node.HasBlock2();
@@ -6355,7 +6355,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS
return BuildProcess(startPos, std::move(source), nullptr, false, {}, false, processStream, settings, {});
}
- const auto& block5 = node.GetBlock5();
+ const auto& block5 = node.GetBlock5();
if (block5.HasBlock5()) {
TSqlExpression expr(Ctx, Mode);
TColumnRefScope scope(Ctx, EColumnRefState::Allow);
@@ -6416,7 +6416,7 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS
if (call.IsExternal())
listCall = true;
- if (block5.HasBlock3()) {
+ if (block5.HasBlock3()) {
with->SetLabel(Id(block5.GetBlock3().GetRule_an_id2(), *this));
}
@@ -6425,21 +6425,21 @@ TSourcePtr TSqlSelect::ProcessCore(const TRule_process_core& node, const TWriteS
return nullptr;
}
- TVector<TSortSpecificationPtr> assumeOrderBy;
+ TVector<TSortSpecificationPtr> assumeOrderBy;
if (block5.HasBlock7()) {
if (!OrderByClause(block5.GetBlock7().GetRule_order_by_clause2(), assumeOrderBy)) {
- return nullptr;
- }
- Ctx.IncrementMonCounter("sql_features", IsColumnsOnly(assumeOrderBy) ? "AssumeOrderBy" : "AssumeOrderByExpr");
- }
-
+ return nullptr;
+ }
+ Ctx.IncrementMonCounter("sql_features", IsColumnsOnly(assumeOrderBy) ? "AssumeOrderBy" : "AssumeOrderByExpr");
+ }
+
return BuildProcess(startPos, std::move(source), with, finalCall.IsExternal(), std::move(args), listCall, processStream, settings, assumeOrderBy);
}
TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSettings& settings, TPosition& selectPos) {
- // REDUCE named_single_source (COMMA named_single_source)* (PRESORT sort_specification_list)?
+ // REDUCE named_single_source (COMMA named_single_source)* (PRESORT sort_specification_list)?
// ON column_list USING ALL? using_call_expr (AS an_id)?
- // (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?
+ // (WHERE expr)? (HAVING expr)? (ASSUME order_by_clause)?
Token(node.GetToken1());
TPosition startPos(Ctx.Pos());
if (!selectPos) {
@@ -6450,34 +6450,34 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet
if (!source) {
return {};
}
- if (node.GetBlock3().size()) {
- TVector<TSourcePtr> sources(1, source);
- for (auto& s: node.GetBlock3()) {
+ if (node.GetBlock3().size()) {
+ TVector<TSourcePtr> sources(1, source);
+ for (auto& s: node.GetBlock3()) {
sources.push_back(NamedSingleSource(s.GetRule_named_single_source2(), true));
- if (!sources.back()) {
- return nullptr;
- }
- }
- auto pos = source->GetPos();
- source = BuildMuxSource(pos, std::move(sources));
- }
+ if (!sources.back()) {
+ return nullptr;
+ }
+ }
+ auto pos = source->GetPos();
+ source = BuildMuxSource(pos, std::move(sources));
+ }
TVector<TSortSpecificationPtr> orderBy;
- if (node.HasBlock4()) {
- if (!SortSpecificationList(node.GetBlock4().GetRule_sort_specification_list2(), orderBy)) {
+ if (node.HasBlock4()) {
+ if (!SortSpecificationList(node.GetBlock4().GetRule_sort_specification_list2(), orderBy)) {
return {};
}
}
TVector<TNodePtr> keys;
- if (!ColumnList(keys, node.GetRule_column_list6())) {
+ if (!ColumnList(keys, node.GetRule_column_list6())) {
return nullptr;
}
- if (node.HasBlock11()) {
+ if (node.HasBlock11()) {
TColumnRefScope scope(Ctx, EColumnRefState::Allow);
TSqlExpression expr(Ctx, Mode);
- TNodePtr where = expr.Build(node.GetBlock11().GetRule_expr2());
+ TNodePtr where = expr.Build(node.GetBlock11().GetRule_expr2());
if (!where || !source->AddFilter(Ctx, where)) {
return nullptr;
}
@@ -6487,10 +6487,10 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet
}
TNodePtr having;
- if (node.HasBlock12()) {
+ if (node.HasBlock12()) {
TColumnRefScope scope(Ctx, EColumnRefState::Allow);
TSqlExpression expr(Ctx, Mode);
- having = expr.Build(node.GetBlock12().GetRule_expr2());
+ having = expr.Build(node.GetBlock12().GetRule_expr2());
if (!having) {
return nullptr;
}
@@ -6525,20 +6525,20 @@ TSourcePtr TSqlSelect::ReduceCore(const TRule_reduce_core& node, const TWriteSet
return {};
}
- if (node.HasBlock10()) {
+ if (node.HasBlock10()) {
udf->SetLabel(Id(node.GetBlock10().GetRule_an_id2(), *this));
}
- const auto reduceMode = node.HasBlock8() ? ReduceMode::ByAll : ReduceMode::ByPartition;
-
- TVector<TSortSpecificationPtr> assumeOrderBy;
- if (node.HasBlock13()) {
- if (!OrderByClause(node.GetBlock13().GetRule_order_by_clause2(), assumeOrderBy)) {
- return nullptr;
- }
- Ctx.IncrementMonCounter("sql_features", IsColumnsOnly(assumeOrderBy) ? "AssumeOrderBy" : "AssumeOrderByExpr");
- }
-
+ const auto reduceMode = node.HasBlock8() ? ReduceMode::ByAll : ReduceMode::ByPartition;
+
+ TVector<TSortSpecificationPtr> assumeOrderBy;
+ if (node.HasBlock13()) {
+ if (!OrderByClause(node.GetBlock13().GetRule_order_by_clause2(), assumeOrderBy)) {
+ return nullptr;
+ }
+ Ctx.IncrementMonCounter("sql_features", IsColumnsOnly(assumeOrderBy) ? "AssumeOrderBy" : "AssumeOrderByExpr");
+ }
+
return BuildReduce(startPos, reduceMode, std::move(source), std::move(orderBy), std::move(keys), std::move(args), udf, having,
settings, assumeOrderBy, listCall);
}
@@ -6653,7 +6653,7 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
Ctx.IncrementMonCounter("sql_features", "WindowClause");
}
- bool assumeSorted = false;
+ bool assumeSorted = false;
TVector<TSortSpecificationPtr> orderBy;
if (node.HasBlock14()) {
auto& orderBlock = node.GetBlock14().GetRule_ext_order_by_clause1();
@@ -6670,13 +6670,13 @@ TSourcePtr TSqlSelect::SelectCore(const TRule_select_core& node, const TWriteSet
return nullptr;
}
- if (!OrderByClause(orderBlock.GetRule_order_by_clause2(), orderBy)) {
+ if (!OrderByClause(orderBlock.GetRule_order_by_clause2(), orderBy)) {
return nullptr;
}
- Ctx.IncrementMonCounter("sql_features", IsColumnsOnly(orderBy)
- ? (assumeSorted ? "AssumeOrderBy" : "OrderBy")
- : (assumeSorted ? "AssumeOrderByExpr" : "OrderByExpr")
- );
+ Ctx.IncrementMonCounter("sql_features", IsColumnsOnly(orderBy)
+ ? (assumeSorted ? "AssumeOrderBy" : "OrderBy")
+ : (assumeSorted ? "AssumeOrderByExpr" : "OrderByExpr")
+ );
if (!NeedPassLimitOrderByToUnderlyingSelect(placement)) {
selectOpOrederBy.swap(orderBy);
@@ -8163,7 +8163,7 @@ private:
bool AlterTableAddChangefeed(const TRule_alter_table_add_changefeed& node, TAlterTableParameters& params);
bool AlterTableAlterChangefeed(const TRule_alter_table_alter_changefeed& node, TAlterTableParameters& params);
void AlterTableDropChangefeed(const TRule_alter_table_drop_changefeed& node, TAlterTableParameters& params);
- TNodePtr PragmaStatement(const TRule_pragma_stmt& stmt, bool& success);
+ TNodePtr PragmaStatement(const TRule_pragma_stmt& stmt, bool& success);
void AddStatementToBlocks(TVector<TNodePtr>& blocks, TNodePtr node);
TNodePtr Build(const TRule_delete_stmt& stmt);
@@ -8315,14 +8315,14 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
switch (altCase) {
case TRule_sql_stmt_core::kAltSqlStmtCore1: {
- bool success = false;
- TNodePtr nodeExpr = PragmaStatement(core.GetAlt_sql_stmt_core1().GetRule_pragma_stmt1(), success);
- if (!success) {
+ bool success = false;
+ TNodePtr nodeExpr = PragmaStatement(core.GetAlt_sql_stmt_core1().GetRule_pragma_stmt1(), success);
+ if (!success) {
return false;
}
- if (nodeExpr) {
+ if (nodeExpr) {
AddStatementToBlocks(blocks, nodeExpr);
- }
+ }
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore2: {
@@ -8345,7 +8345,7 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
if (!nodeExpr) {
return false;
}
- TVector<TNodePtr> nodes;
+ TVector<TNodePtr> nodes;
auto subquery = nodeExpr->GetSource();
if (subquery) {
const auto alias = Ctx.MakeName("subquerynode");
@@ -8353,24 +8353,24 @@ bool TSqlQuery::Statement(TVector<TNodePtr>& blocks, const TRule_sql_stmt_core&
blocks.push_back(BuildSubquery(subquery, alias,
Mode == NSQLTranslation::ESqlMode::SUBQUERY, names.size() == 1 ? -1 : names.size(), Ctx.Scoped));
blocks.back()->SetLabel(ref);
-
- for (size_t i = 0; i < names.size(); ++i) {
- nodes.push_back(BuildSubqueryRef(blocks.back(), ref, names.size() == 1 ? -1 : i));
- }
- } else {
- if (names.size() > 1) {
+
+ for (size_t i = 0; i < names.size(); ++i) {
+ nodes.push_back(BuildSubqueryRef(blocks.back(), ref, names.size() == 1 ? -1 : i));
+ }
+ } else {
+ if (names.size() > 1) {
auto tupleRes = BuildTupleResult(nodeExpr, names.size());
- for (size_t i = 0; i < names.size(); ++i) {
+ for (size_t i = 0; i < names.size(); ++i) {
nodes.push_back(nodeExpr->Y("Nth", tupleRes, nodeExpr->Q(ToString(i))));
- }
- } else {
- nodes.push_back(std::move(nodeExpr));
- }
+ }
+ } else {
+ nodes.push_back(std::move(nodeExpr));
+ }
}
-
- for (size_t i = 0; i < names.size(); ++i) {
+
+ for (size_t i = 0; i < names.size(); ++i) {
PushNamedNode(names[i].Pos, names[i].Name, nodes[i]);
- }
+ }
break;
}
case TRule_sql_stmt_core::kAltSqlStmtCore4: {
@@ -9131,7 +9131,7 @@ void TSqlQuery::AlterTableDropChangefeed(const TRule_alter_table_drop_changefeed
params.DropChangefeeds.emplace_back(IdEx(node.GetRule_an_id3(), *this));
}
-TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success) {
+TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success) {
success = false;
const TString& prefix = OptIdPrefixAsStr(stmt.GetRule_opt_id_prefix_or_type2(), *this);
const TString& lowerPrefix = to_lower(prefix);
@@ -9141,12 +9141,12 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (!normalizeError.Empty()) {
Error() << normalizeError->Message;
Ctx.IncrementMonCounter("sql_errors", "NormalizePragmaError");
- return {};
+ return {};
}
TVector<TDeferredAtom> values;
TVector<const TRule_pragma_value*> pragmaValues;
- bool pragmaValueDefault = false;
+ bool pragmaValueDefault = false;
if (stmt.GetBlock4().HasAlt1()) {
pragmaValues.push_back(&stmt.GetBlock4().GetAlt1().GetRule_pragma_value2());
}
@@ -9169,13 +9169,13 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
}
values.push_back(TDeferredAtom(Ctx.Pos(), parsed->Content));
- }
- else if (pragmaValue->HasAlt_pragma_value2()
- && pragmaValue->GetAlt_pragma_value2().GetRule_id1().HasAlt_id2()
+ }
+ else if (pragmaValue->HasAlt_pragma_value2()
+ && pragmaValue->GetAlt_pragma_value2().GetRule_id1().HasAlt_id2()
&& "default" == Id(pragmaValue->GetAlt_pragma_value2().GetRule_id1(), *this))
- {
- pragmaValueDefault = true;
- }
+ {
+ pragmaValueDefault = true;
+ }
else if (withConfigure && pragmaValue->HasAlt_pragma_value5()) {
TString bindName;
if (!NamedNodeImpl(pragmaValue->GetAlt_pragma_value5().GetRule_bind_parameter1(), bindName, *this)) {
@@ -9192,7 +9192,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
} else {
Error() << "Expected string" << (withConfigure ? ", named parameter" : "") << " or 'default' keyword as pragma value for pragma: " << pragma;
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
}
@@ -9216,7 +9216,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (values.size() != 1 || pragmaValueDefault) {
Error() << "Expected file alias as pragma value";
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
Ctx.IncrementMonCounter("sql_pragma", "udf");
@@ -9251,7 +9251,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (values.size() != 2U || pragmaValueDefault) {
Error() << "Expected file alias and url as pragma values";
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
Ctx.IncrementMonCounter("sql_pragma", "file");
@@ -9290,9 +9290,9 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
Ctx.Libraries[alias]=file;
Ctx.IncrementMonCounter("sql_pragma", "library");
- } else if (normalizedPragma == "directread") {
- Ctx.PragmaDirectRead = true;
- Ctx.IncrementMonCounter("sql_pragma", "DirectRead");
+ } else if (normalizedPragma == "directread") {
+ Ctx.PragmaDirectRead = true;
+ Ctx.IncrementMonCounter("sql_pragma", "DirectRead");
} else if (normalizedPragma == "equijoin") {
Ctx.IncrementMonCounter("sql_pragma", "EquiJoin");
} else if (normalizedPragma == "autocommit") {
@@ -9305,7 +9305,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (values.size() == 1 || values.size() == 2) {
if (!values.front().GetLiteral(value, Ctx)) {
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
if (values.size() == 2) {
@@ -9317,13 +9317,13 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
}
if (!Ctx.SetPathPrefix(value, arg)) {
- return {};
+ return {};
}
} else {
Error() << "Expected path prefix or tuple of (Provider, PathPrefix) or"
<< " (Cluster, PathPrefix) as pragma value";
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
Ctx.IncrementMonCounter("sql_pragma", "PathPrefix");
@@ -9331,14 +9331,14 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.PragmaGroupByLimit)) {
Error() << "Expected unsigned integer literal as a single argument for: " << pragma;
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
Ctx.IncrementMonCounter("sql_pragma", "GroupByLimit");
} else if (normalizedPragma == "groupbycubelimit") {
if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.PragmaGroupByCubeLimit)) {
Error() << "Expected unsigned integer literal as a single argument for: " << pragma;
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
Ctx.IncrementMonCounter("sql_pragma", "GroupByCubeLimit");
} else if (normalizedPragma == "simplecolumns") {
@@ -9442,9 +9442,9 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
return {};
}
Ctx.IncrementMonCounter("sql_pragma", "ClassicDivision");
- } else if (normalizedPragma == "disableunordered") {
- Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_PRAGMA)
- << "Use of deprecated DisableUnordered pragma. It will be dropped soon";
+ } else if (normalizedPragma == "disableunordered") {
+ Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_PRAGMA)
+ << "Use of deprecated DisableUnordered pragma. It will be dropped soon";
} else if (normalizedPragma == "pullupflatmapoverjoin") {
Ctx.PragmaPullUpFlatMapOverJoin = true;
Ctx.IncrementMonCounter("sql_pragma", "PullUpFlatMapOverJoin");
@@ -9457,9 +9457,9 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
} else if (normalizedPragma == "warnunnamedcolumns") {
Ctx.WarnUnnamedColumns = true;
Ctx.IncrementMonCounter("sql_pragma", "WarnUnnamedColumns");
- } else if (normalizedPragma == "discoverymode") {
- Ctx.DiscoveryMode = true;
- Ctx.IncrementMonCounter("sql_pragma", "DiscoveryMode");
+ } else if (normalizedPragma == "discoverymode") {
+ Ctx.DiscoveryMode = true;
+ Ctx.IncrementMonCounter("sql_pragma", "DiscoveryMode");
} else if (normalizedPragma == "enablesystemcolumns") {
if (values.size() != 1 || !values[0].GetLiteral() || !TryFromString(*values[0].GetLiteral(), Ctx.EnableSystemColumns)) {
Error() << "Expected boolean literal as a single argument for: " << pragma;
@@ -9522,7 +9522,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
return {};
}
- Ctx.IncrementMonCounter("sql_pragma", "RegexUseRe2");
+ Ctx.IncrementMonCounter("sql_pragma", "RegexUseRe2");
} else if (normalizedPragma == "jsonqueryreturnsjsondocument") {
Ctx.JsonQueryReturnsJsonDocument = true;
Ctx.IncrementMonCounter("sql_pragma", "JsonQueryReturnsJsonDocument");
@@ -9599,7 +9599,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
} else {
Error() << "Unknown pragma: " << pragma;
Ctx.IncrementMonCounter("sql_errors", "UnknownPragma");
- return {};
+ return {};
}
} else {
if (lowerPrefix == "yson") {
@@ -9609,8 +9609,8 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
return {};
}
if (normalizedPragma == "fast") {
- Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_PRAGMA)
- << "Use of deprecated yson.Fast pragma. It will be dropped soon";
+ Ctx.Warning(Ctx.Pos(), TIssuesIds::YQL_DEPRECATED_PRAGMA)
+ << "Use of deprecated yson.Fast pragma. It will be dropped soon";
success = true;
return {};
} else if (normalizedPragma == "autoconvert") {
@@ -9654,7 +9654,7 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
Error() << "Unknown pragma prefix: " << prefix << ", please use cluster name or one of provider " <<
JoinRange(", ", Providers.cbegin(), Providers.cend());
Ctx.IncrementMonCounter("sql_errors", "UnknownPragma");
- return {};
+ return {};
}
}
@@ -9662,22 +9662,22 @@ TNodePtr TSqlQuery::PragmaStatement(const TRule_pragma_stmt& stmt, bool& success
if (values.size() > 1) {
Error() << "Expected at most one value in the pragma";
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
} else {
- if (pragmaValueDefault || values.size() < 1) {
+ if (pragmaValueDefault || values.size() < 1) {
Error() << "Expected at least one value in the pragma";
Ctx.IncrementMonCounter("sql_errors", "BadPragmaValue");
- return {};
+ return {};
}
}
success = true;
Ctx.IncrementMonCounter("sql_pragma", pragma);
- return BuildPragma(Ctx.Pos(), lowerPrefix, normalizedPragma, values, pragmaValueDefault);
+ return BuildPragma(Ctx.Pos(), lowerPrefix, normalizedPragma, values, pragmaValueDefault);
}
success = true;
- return {};
+ return {};
}
TNodePtr TSqlQuery::Build(const TRule_delete_stmt& stmt) {
diff --git a/ydb/library/yql/sql/v1/sql.h b/ydb/library/yql/sql/v1/sql.h
index 9c7dc676c6..7550cb88d0 100644
--- a/ydb/library/yql/sql/v1/sql.h
+++ b/ydb/library/yql/sql/v1/sql.h
@@ -5,9 +5,9 @@
#include <ydb/library/yql/public/issue/yql_warning.h>
#include <ydb/library/yql/public/issue/yql_issue_manager.h>
#include <ydb/library/yql/sql/settings/translation_settings.h>
-
+
#include <google/protobuf/message.h>
-
+
namespace NSQLTranslation {
struct TTranslationSettings;
}
diff --git a/ydb/library/yql/sql/v1/sql_ut.cpp b/ydb/library/yql/sql/v1/sql_ut.cpp
index 85b6e0dfcd..2922b015f7 100644
--- a/ydb/library/yql/sql/v1/sql_ut.cpp
+++ b/ydb/library/yql/sql/v1/sql_ut.cpp
@@ -28,7 +28,7 @@ NYql::TAstParseResult SqlToYqlWithMode(const TString& query, NSQLTranslation::ES
EDebugOutput debug = EDebugOutput::None, bool ansiLexer = false, NSQLTranslation::TTranslationSettings settings = {})
{
google::protobuf::Arena arena;
- const auto service = provider ? provider : TString(NYql::YtProviderName);
+ const auto service = provider ? provider : TString(NYql::YtProviderName);
const TString cluster = "plato";
settings.ClusterMapping[cluster] = service;
settings.ClusterMapping["hahn"] = NYql::YtProviderName;
@@ -858,7 +858,7 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
TWordCountHive elementStat = {{TString("Kikimr.PushData"), 0}};
VerifyProgram(res, elementStat, verifyLine);
- UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]);
+ UNIT_ASSERT_VALUES_EQUAL(1, elementStat["Kikimr.PushData"]);
}
Y_UNIT_TEST(ProcessUserTypeAuth) {
@@ -1106,12 +1106,12 @@ Y_UNIT_TEST_SUITE(SqlParsingOnly) {
UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t.x").IsOk());
UNIT_ASSERT(SqlToYql("select * from plato.Input as t flatten by t -- same as flatten by t.t").IsOk());
}
-
- Y_UNIT_TEST(DiscoveryMode) {
- UNIT_ASSERT(SqlToYqlWithMode("insert into plato.Output select * from plato.Input", NSQLTranslation::ESqlMode::DISCOVERY).IsOk());
- UNIT_ASSERT(SqlToYqlWithMode("select * from plato.concat(Input1, Input2)", NSQLTranslation::ESqlMode::DISCOVERY).IsOk());
- UNIT_ASSERT(SqlToYqlWithMode("select * from plato.each(AsList(\"Input1\", \"Input2\"))", NSQLTranslation::ESqlMode::DISCOVERY).IsOk());
- }
+
+ Y_UNIT_TEST(DiscoveryMode) {
+ UNIT_ASSERT(SqlToYqlWithMode("insert into plato.Output select * from plato.Input", NSQLTranslation::ESqlMode::DISCOVERY).IsOk());
+ UNIT_ASSERT(SqlToYqlWithMode("select * from plato.concat(Input1, Input2)", NSQLTranslation::ESqlMode::DISCOVERY).IsOk());
+ UNIT_ASSERT(SqlToYqlWithMode("select * from plato.each(AsList(\"Input1\", \"Input2\"))", NSQLTranslation::ESqlMode::DISCOVERY).IsOk());
+ }
Y_UNIT_TEST(CubeWithAutoGeneratedLikeColumnName) {
UNIT_ASSERT(SqlToYql("select key,subkey,group from plato.Input group by cube(key,subkey,group)").IsOk());
@@ -1671,7 +1671,7 @@ Y_UNIT_TEST_SUITE(SqlToYQLErrors) {
UNIT_ASSERT(!res.Root);
TString a1 = Err2Str(res);
- TString a2(R"foo(<main>:1:14: Error: Unexpected character 'с' (Unicode character <1089>) : cannot match to any predicted input...
+ TString a2(R"foo(<main>:1:14: Error: Unexpected character 'с' (Unicode character <1089>) : cannot match to any predicted input...
<main>:1:15: Error: Unexpected character : cannot match to any predicted input...
@@ -2707,28 +2707,28 @@ select FormatType($f());
UNIT_ASSERT(!res.Root);
UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:5: Error: Unknown cluster: PlatO\n");
}
-
- Y_UNIT_TEST(DiscoveryModeForbidden) {
- NYql::TAstParseResult res = SqlToYqlWithMode("insert into plato.Output select * from plato.range(\"\", Input1, Input4)", NSQLTranslation::ESqlMode::DISCOVERY);
- UNIT_ASSERT(!res.Root);
- UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: range is not allowed in Discovery mode, code: 4600\n");
-
- res = SqlToYqlWithMode("insert into plato.Output select * from plato.like(\"\", \"Input%\")", NSQLTranslation::ESqlMode::DISCOVERY);
- UNIT_ASSERT(!res.Root);
- UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: like is not allowed in Discovery mode, code: 4600\n");
-
- res = SqlToYqlWithMode("insert into plato.Output select * from plato.regexp(\"\", \"Input.\")", NSQLTranslation::ESqlMode::DISCOVERY);
- UNIT_ASSERT(!res.Root);
- UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: regexp is not allowed in Discovery mode, code: 4600\n");
-
- res = SqlToYqlWithMode("insert into plato.Output select * from plato.filter(\"\", ($name) -> { return find($name, \"Input\") is not null; })", NSQLTranslation::ESqlMode::DISCOVERY);
- UNIT_ASSERT(!res.Root);
- UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: filter is not allowed in Discovery mode, code: 4600\n");
-
- res = SqlToYqlWithMode("select Path from plato.folder(\"\") where Type == \"table\"", NSQLTranslation::ESqlMode::DISCOVERY);
- UNIT_ASSERT(!res.Root);
- UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: folder is not allowed in Discovery mode, code: 4600\n");
- }
+
+ Y_UNIT_TEST(DiscoveryModeForbidden) {
+ NYql::TAstParseResult res = SqlToYqlWithMode("insert into plato.Output select * from plato.range(\"\", Input1, Input4)", NSQLTranslation::ESqlMode::DISCOVERY);
+ UNIT_ASSERT(!res.Root);
+ UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: range is not allowed in Discovery mode, code: 4600\n");
+
+ res = SqlToYqlWithMode("insert into plato.Output select * from plato.like(\"\", \"Input%\")", NSQLTranslation::ESqlMode::DISCOVERY);
+ UNIT_ASSERT(!res.Root);
+ UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: like is not allowed in Discovery mode, code: 4600\n");
+
+ res = SqlToYqlWithMode("insert into plato.Output select * from plato.regexp(\"\", \"Input.\")", NSQLTranslation::ESqlMode::DISCOVERY);
+ UNIT_ASSERT(!res.Root);
+ UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: regexp is not allowed in Discovery mode, code: 4600\n");
+
+ res = SqlToYqlWithMode("insert into plato.Output select * from plato.filter(\"\", ($name) -> { return find($name, \"Input\") is not null; })", NSQLTranslation::ESqlMode::DISCOVERY);
+ UNIT_ASSERT(!res.Root);
+ UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:40: Error: filter is not allowed in Discovery mode, code: 4600\n");
+
+ res = SqlToYqlWithMode("select Path from plato.folder(\"\") where Type == \"table\"", NSQLTranslation::ESqlMode::DISCOVERY);
+ UNIT_ASSERT(!res.Root);
+ UNIT_ASSERT_NO_DIFF(Err2Str(res), "<main>:1:18: Error: folder is not allowed in Discovery mode, code: 4600\n");
+ }
Y_UNIT_TEST(YsonFuncWithoutArgs) {
UNIT_ASSERT(SqlToYql("SELECT Yson::SerializeText(Yson::From());").IsOk());
diff --git a/ydb/library/yql/udfs/common/hyperscan/ya.make b/ydb/library/yql/udfs/common/hyperscan/ya.make
index f8bb10662d..695e83afe4 100644
--- a/ydb/library/yql/udfs/common/hyperscan/ya.make
+++ b/ydb/library/yql/udfs/common/hyperscan/ya.make
@@ -1,27 +1,27 @@
OWNER(g:yql g:yql_ydb_core)
-
+
IF (OS_LINUX AND CLANG)
- YQL_UDF(hyperscan_udf)
+ YQL_UDF(hyperscan_udf)
- YQL_ABI_VERSION(
- 2
+ YQL_ABI_VERSION(
+ 2
23
- 0
- )
+ 0
+ )
- SRCS(
- hyperscan_udf.cpp
- )
+ SRCS(
+ hyperscan_udf.cpp
+ )
- PEERDIR(
+ PEERDIR(
library/cpp/regex/hyperscan
library/cpp/regex/pcre
- )
+ )
- END()
+ END()
-ELSE()
- LIBRARY()
- END()
+ELSE()
+ LIBRARY()
+ END()
ENDIF()
diff --git a/ydb/library/yql/udfs/common/math/math_ir.cpp b/ydb/library/yql/udfs/common/math/math_ir.cpp
index a63968dbe0..6ba63980c1 100644
--- a/ydb/library/yql/udfs/common/math/math_ir.cpp
+++ b/ydb/library/yql/udfs/common/math/math_ir.cpp
@@ -1,5 +1,5 @@
-#define LLVM_BC
-
-#include "math_ir.h"
-
-#include <util/generic/ymath.cpp>
+#define LLVM_BC
+
+#include "math_ir.h"
+
+#include <util/generic/ymath.cpp>
diff --git a/ydb/library/yql/udfs/common/math/math_ir.h b/ydb/library/yql/udfs/common/math/math_ir.h
index 2386be96b9..85eddfbe3c 100644
--- a/ydb/library/yql/udfs/common/math/math_ir.h
+++ b/ydb/library/yql/udfs/common/math/math_ir.h
@@ -1,118 +1,118 @@
-#pragma once
-
+#pragma once
+
#include <ydb/library/yql/udfs/common/math/lib/round.h>
#include <ydb/library/yql/public/udf/udf_value.h>
-
-#include <util/generic/ymath.h>
-#include <util/system/compiler.h>
-
-#include <math.h>
-
+
+#include <util/generic/ymath.h>
+#include <util/system/compiler.h>
+
+#include <math.h>
+
namespace NYql {
-namespace NUdf {
-
-#define CONST_FUNCS(XX) \
- XX(Pi, M_PI) \
- XX(E, M_E)
-
-#define SINGLE_ARG_FUNCS(XX) \
- XX(Abs, Abs) \
- XX(Acos, acos) \
- XX(Asin, asin) \
- XX(Asinh, asin) \
- XX(Atan, atan) \
- XX(Cbrt, cbrt) \
- XX(Ceil, ceil) \
- XX(Cos, cos) \
- XX(Cosh, cosh) \
- XX(Erf, Erf) \
- XX(Exp, exp) \
- XX(Exp2, Exp2) \
- XX(Fabs, fabs) \
- XX(Floor, std::floor) \
- XX(Lgamma, LogGamma) \
- XX(Rint, rint) \
- XX(Sin, sin) \
- XX(Sinh, sinh) \
- XX(Sqrt, sqrt) \
- XX(Tan, tan) \
- XX(Tanh, tanh) \
- XX(Tgamma, tgamma) \
- XX(Trunc, trunc) \
- XX(IsFinite, std::isfinite) \
- XX(IsInf, std::isinf) \
- XX(IsNaN, std::isnan)
-
-#define TWO_ARGS_FUNCS(XX) \
- XX(Atan2, atan2, double) \
- XX(Fmod, fmod, double) \
- XX(Hypot, hypot, double) \
- XX(Remainder, remainder, double) \
- XX(Pow, pow, double) \
- XX(Ldexp, ldexp, int)
-
-#define POSITIVE_SINGLE_ARG_FUNCS(XX) \
- XX(Log, log) \
- XX(Log2, Log2) \
- XX(Log10, log10)
-
-
-#define CONST_IMPL(name, cnst) \
- extern "C" UDF_ALWAYS_INLINE \
- void name##IR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* /*args*/) {\
- *result = TUnboxedValuePod(cnst); \
- }
-
-#define SINGLE_ARG_IMPL(name, func) \
- extern "C" UDF_ALWAYS_INLINE \
- void name##IR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { \
- *result = TUnboxedValuePod(func(args[0].Get<double>())); \
- }
-
-#define TWO_ARGS_IMPL(name, func, secondType) \
- extern "C" UDF_ALWAYS_INLINE \
- void name##IR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { \
- *result = TUnboxedValuePod(func(args[0].Get<double>(), args[1].Get<secondType>())); \
- }
-
-#define POSITIVE_SINGLE_ARG_IMPL(name, func) \
- extern "C" UDF_ALWAYS_INLINE \
- void name##IR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { \
- double input = args[0].Get<double>(); \
- if (input > 0) { \
- *result = TUnboxedValuePod(func(input)); \
- } else { \
- *result = TUnboxedValuePod(static_cast<double>(NAN)); \
- } \
- }
-
-CONST_FUNCS(CONST_IMPL)
-SINGLE_ARG_FUNCS(SINGLE_ARG_IMPL)
-TWO_ARGS_FUNCS(TWO_ARGS_IMPL)
-POSITIVE_SINGLE_ARG_FUNCS(POSITIVE_SINGLE_ARG_IMPL)
-
-extern "C" UDF_ALWAYS_INLINE
-void SigmoidIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) {
- *result = TUnboxedValuePod(1. / (1. + exp(-args[0].Get<double>())));
-}
-
-
-extern "C" UDF_ALWAYS_INLINE
-void FuzzyEqualsIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) {
- if (!args[2]) {
- *result = TUnboxedValuePod(FuzzyEquals(args[0].Get<double>(), args[1].Get<double>()));
- } else {
- const double eps = args[2].Get<double>();
- *result = TUnboxedValuePod(FuzzyEquals(args[0].Get<double>(), args[1].Get<double>(), eps));
- }
-}
-
-extern "C" UDF_ALWAYS_INLINE
-void RoundIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) {
+namespace NUdf {
+
+#define CONST_FUNCS(XX) \
+ XX(Pi, M_PI) \
+ XX(E, M_E)
+
+#define SINGLE_ARG_FUNCS(XX) \
+ XX(Abs, Abs) \
+ XX(Acos, acos) \
+ XX(Asin, asin) \
+ XX(Asinh, asin) \
+ XX(Atan, atan) \
+ XX(Cbrt, cbrt) \
+ XX(Ceil, ceil) \
+ XX(Cos, cos) \
+ XX(Cosh, cosh) \
+ XX(Erf, Erf) \
+ XX(Exp, exp) \
+ XX(Exp2, Exp2) \
+ XX(Fabs, fabs) \
+ XX(Floor, std::floor) \
+ XX(Lgamma, LogGamma) \
+ XX(Rint, rint) \
+ XX(Sin, sin) \
+ XX(Sinh, sinh) \
+ XX(Sqrt, sqrt) \
+ XX(Tan, tan) \
+ XX(Tanh, tanh) \
+ XX(Tgamma, tgamma) \
+ XX(Trunc, trunc) \
+ XX(IsFinite, std::isfinite) \
+ XX(IsInf, std::isinf) \
+ XX(IsNaN, std::isnan)
+
+#define TWO_ARGS_FUNCS(XX) \
+ XX(Atan2, atan2, double) \
+ XX(Fmod, fmod, double) \
+ XX(Hypot, hypot, double) \
+ XX(Remainder, remainder, double) \
+ XX(Pow, pow, double) \
+ XX(Ldexp, ldexp, int)
+
+#define POSITIVE_SINGLE_ARG_FUNCS(XX) \
+ XX(Log, log) \
+ XX(Log2, Log2) \
+ XX(Log10, log10)
+
+
+#define CONST_IMPL(name, cnst) \
+ extern "C" UDF_ALWAYS_INLINE \
+ void name##IR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* /*args*/) {\
+ *result = TUnboxedValuePod(cnst); \
+ }
+
+#define SINGLE_ARG_IMPL(name, func) \
+ extern "C" UDF_ALWAYS_INLINE \
+ void name##IR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { \
+ *result = TUnboxedValuePod(func(args[0].Get<double>())); \
+ }
+
+#define TWO_ARGS_IMPL(name, func, secondType) \
+ extern "C" UDF_ALWAYS_INLINE \
+ void name##IR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { \
+ *result = TUnboxedValuePod(func(args[0].Get<double>(), args[1].Get<secondType>())); \
+ }
+
+#define POSITIVE_SINGLE_ARG_IMPL(name, func) \
+ extern "C" UDF_ALWAYS_INLINE \
+ void name##IR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) { \
+ double input = args[0].Get<double>(); \
+ if (input > 0) { \
+ *result = TUnboxedValuePod(func(input)); \
+ } else { \
+ *result = TUnboxedValuePod(static_cast<double>(NAN)); \
+ } \
+ }
+
+CONST_FUNCS(CONST_IMPL)
+SINGLE_ARG_FUNCS(SINGLE_ARG_IMPL)
+TWO_ARGS_FUNCS(TWO_ARGS_IMPL)
+POSITIVE_SINGLE_ARG_FUNCS(POSITIVE_SINGLE_ARG_IMPL)
+
+extern "C" UDF_ALWAYS_INLINE
+void SigmoidIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) {
+ *result = TUnboxedValuePod(1. / (1. + exp(-args[0].Get<double>())));
+}
+
+
+extern "C" UDF_ALWAYS_INLINE
+void FuzzyEqualsIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) {
+ if (!args[2]) {
+ *result = TUnboxedValuePod(FuzzyEquals(args[0].Get<double>(), args[1].Get<double>()));
+ } else {
+ const double eps = args[2].Get<double>();
+ *result = TUnboxedValuePod(FuzzyEquals(args[0].Get<double>(), args[1].Get<double>(), eps));
+ }
+}
+
+extern "C" UDF_ALWAYS_INLINE
+void RoundIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) {
const double val = NMathUdf::RoundToDecimal<long double>(args[0].Get<double>(), args[1].GetOrDefault<int>(0));
- *result = TUnboxedValuePod(val);
-}
-
+ *result = TUnboxedValuePod(val);
+}
+
extern "C" UDF_ALWAYS_INLINE
void ModIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueBuilder* /*valueBuilder*/, const TUnboxedValuePod* args) {
const auto val = NMathUdf::Mod(args[0].Get<i64>(), args[1].Get<i64>());
@@ -125,5 +125,5 @@ void RemIR(const IBoxedValue* /*pThis*/, TUnboxedValuePod* result, const IValueB
*result = val ? TUnboxedValuePod(*val) : TUnboxedValuePod();
}
-} // NUdf
+} // NUdf
} // NYql
diff --git a/ydb/library/yql/udfs/common/math/math_udf.cpp b/ydb/library/yql/udfs/common/math/math_udf.cpp
index 04aaa29fea..ce019a8b91 100644
--- a/ydb/library/yql/udfs/common/math/math_udf.cpp
+++ b/ydb/library/yql/udfs/common/math/math_udf.cpp
@@ -1,61 +1,61 @@
-#include "math_ir.h"
-
+#include "math_ir.h"
+
#include <ydb/library/yql/public/udf/udf_helpers.h>
-#define MATH_UDF_MAP(XX, XXL) \
- XX(Pi, double(), ;) \
- XX(E, double(), ;) \
- XX(Abs, double(TAutoMap<double>), ;) \
- XX(Acos, double(TAutoMap<double>), ;) \
- XX(Asin, double(TAutoMap<double>), ;) \
- XX(Asinh, double(TAutoMap<double>), ;) \
- XX(Atan, double(TAutoMap<double>), ;) \
- XX(Cbrt, double(TAutoMap<double>), ;) \
- XX(Ceil, double(TAutoMap<double>), ;) \
- XX(Cos, double(TAutoMap<double>), ;) \
- XX(Cosh, double(TAutoMap<double>), ;) \
- XX(Erf, double(TAutoMap<double>), ;) \
- XX(Exp, double(TAutoMap<double>), ;) \
- XX(Exp2, double(TAutoMap<double>), ;) \
- XX(Fabs, double(TAutoMap<double>), ;) \
- XX(Floor, double(TAutoMap<double>), ;) \
- XX(Lgamma, double(TAutoMap<double>), ;) \
- XX(Rint, double(TAutoMap<double>), ;) \
- XX(Sin, double(TAutoMap<double>), ;) \
- XX(Sinh, double(TAutoMap<double>), ;) \
- XX(Sqrt, double(TAutoMap<double>), ;) \
- XX(Tan, double(TAutoMap<double>), ;) \
- XX(Tanh, double(TAutoMap<double>), ;) \
- XX(Tgamma, double(TAutoMap<double>), ;) \
- XX(Trunc, double(TAutoMap<double>), ;) \
- XX(Log, double(TAutoMap<double>), ;) \
- XX(Log2, double(TAutoMap<double>), ;) \
- XX(Log10, double(TAutoMap<double>), ;) \
- XX(Atan2, double(TAutoMap<double>, TAutoMap<double>), ;) \
- XX(Fmod, double(TAutoMap<double>, TAutoMap<double>), ;) \
- XX(Hypot, double(TAutoMap<double>, TAutoMap<double>), ;) \
- XX(Remainder, double(TAutoMap<double>, TAutoMap<double>), ;) \
- XX(Pow, double(TAutoMap<double>, TAutoMap<double>), ;) \
- XX(Ldexp, double(TAutoMap<double>, TAutoMap<int>), ;) \
- XX(IsFinite, bool(TAutoMap<double>), ;) \
- XX(IsInf, bool(TAutoMap<double>), ;) \
- XX(IsNaN, bool(TAutoMap<double>), ;) \
- XX(Sigmoid, double(TAutoMap<double>), ;) \
- XX(FuzzyEquals, bool(TAutoMap<double>, TAutoMap<double>, TEpsilon), builder.OptionalArgs(1)) \
+#define MATH_UDF_MAP(XX, XXL) \
+ XX(Pi, double(), ;) \
+ XX(E, double(), ;) \
+ XX(Abs, double(TAutoMap<double>), ;) \
+ XX(Acos, double(TAutoMap<double>), ;) \
+ XX(Asin, double(TAutoMap<double>), ;) \
+ XX(Asinh, double(TAutoMap<double>), ;) \
+ XX(Atan, double(TAutoMap<double>), ;) \
+ XX(Cbrt, double(TAutoMap<double>), ;) \
+ XX(Ceil, double(TAutoMap<double>), ;) \
+ XX(Cos, double(TAutoMap<double>), ;) \
+ XX(Cosh, double(TAutoMap<double>), ;) \
+ XX(Erf, double(TAutoMap<double>), ;) \
+ XX(Exp, double(TAutoMap<double>), ;) \
+ XX(Exp2, double(TAutoMap<double>), ;) \
+ XX(Fabs, double(TAutoMap<double>), ;) \
+ XX(Floor, double(TAutoMap<double>), ;) \
+ XX(Lgamma, double(TAutoMap<double>), ;) \
+ XX(Rint, double(TAutoMap<double>), ;) \
+ XX(Sin, double(TAutoMap<double>), ;) \
+ XX(Sinh, double(TAutoMap<double>), ;) \
+ XX(Sqrt, double(TAutoMap<double>), ;) \
+ XX(Tan, double(TAutoMap<double>), ;) \
+ XX(Tanh, double(TAutoMap<double>), ;) \
+ XX(Tgamma, double(TAutoMap<double>), ;) \
+ XX(Trunc, double(TAutoMap<double>), ;) \
+ XX(Log, double(TAutoMap<double>), ;) \
+ XX(Log2, double(TAutoMap<double>), ;) \
+ XX(Log10, double(TAutoMap<double>), ;) \
+ XX(Atan2, double(TAutoMap<double>, TAutoMap<double>), ;) \
+ XX(Fmod, double(TAutoMap<double>, TAutoMap<double>), ;) \
+ XX(Hypot, double(TAutoMap<double>, TAutoMap<double>), ;) \
+ XX(Remainder, double(TAutoMap<double>, TAutoMap<double>), ;) \
+ XX(Pow, double(TAutoMap<double>, TAutoMap<double>), ;) \
+ XX(Ldexp, double(TAutoMap<double>, TAutoMap<int>), ;) \
+ XX(IsFinite, bool(TAutoMap<double>), ;) \
+ XX(IsInf, bool(TAutoMap<double>), ;) \
+ XX(IsNaN, bool(TAutoMap<double>), ;) \
+ XX(Sigmoid, double(TAutoMap<double>), ;) \
+ XX(FuzzyEquals, bool(TAutoMap<double>, TAutoMap<double>, TEpsilon), builder.OptionalArgs(1)) \
XX(Mod, TOptional<i64>(TAutoMap<i64>, i64), ;) \
XX(Rem, TOptional<i64>(TAutoMap<i64>, i64), ;) \
- XXL(Round, double(TAutoMap<double>, TPrecision), builder.OptionalArgs(1))
+ XXL(Round, double(TAutoMap<double>, TPrecision), builder.OptionalArgs(1))
-#define MATH_UDF_IMPL(name, signature, options) \
- UDF_IMPL(T##name, builder.SimpleSignature<signature>(); options;, ;, ;, "/llvm_bc/Math", #name "IR") { \
- TUnboxedValuePod res; \
- name##IR(this, &res, valueBuilder, args); \
- return res; \
+#define MATH_UDF_IMPL(name, signature, options) \
+ UDF_IMPL(T##name, builder.SimpleSignature<signature>(); options;, ;, ;, "/llvm_bc/Math", #name "IR") { \
+ TUnboxedValuePod res; \
+ name##IR(this, &res, valueBuilder, args); \
+ return res; \
}
-#define REGISTER_MATH_UDF(udfName, ...) T##udfName,
-#define REGISTER_MATH_UDF_LAST(udfName, ...) T##udfName
+#define REGISTER_MATH_UDF(udfName, ...) T##udfName,
+#define REGISTER_MATH_UDF_LAST(udfName, ...) T##udfName
using namespace NKikimr;
using namespace NUdf;
@@ -67,10 +67,10 @@ namespace {
extern const char precision[] = "Precision";
using TPrecision = TNamedArg<int, precision>;
- MATH_UDF_MAP(MATH_UDF_IMPL, MATH_UDF_IMPL)
+ MATH_UDF_MAP(MATH_UDF_IMPL, MATH_UDF_IMPL)
SIMPLE_MODULE(TMathModule,
- MATH_UDF_MAP(REGISTER_MATH_UDF, REGISTER_MATH_UDF_LAST))
+ MATH_UDF_MAP(REGISTER_MATH_UDF, REGISTER_MATH_UDF_LAST))
}
REGISTER_MODULES(TMathModule)
diff --git a/ydb/library/yql/udfs/common/math/ya.make b/ydb/library/yql/udfs/common/math/ya.make
index 7aaedf9619..468a44d33b 100644
--- a/ydb/library/yql/udfs/common/math/ya.make
+++ b/ydb/library/yql/udfs/common/math/ya.make
@@ -2,7 +2,7 @@ YQL_UDF(math_udf)
YQL_ABI_VERSION(
2
- 16
+ 16
0
)
@@ -12,54 +12,54 @@ SRCS(
math_udf.cpp
)
-LLVM_BC(
- math_ir.cpp
- NAME Math
- SYMBOLS
- PiIR
- EIR
- AbsIR
- AcosIR
- AsinIR
- AsinhIR
- AtanIR
- CbrtIR
- CeilIR
- CosIR
- CoshIR
- ErfIR
- ExpIR
- Exp2IR
- FabsIR
- FloorIR
- LgammaIR
- RintIR
- SinIR
- SinhIR
- SqrtIR
- TanIR
- TanhIR
- TgammaIR
- TruncIR
- IsFiniteIR
- IsInfIR
- IsNaNIR
- Atan2IR
- FmodIR
- HypotIR
- RemainderIR
- PowIR
- LdexpIR
- LogIR
- Log2IR
- Log10IR
- SigmoidIR
- FuzzyEqualsIR
- RoundIR
+LLVM_BC(
+ math_ir.cpp
+ NAME Math
+ SYMBOLS
+ PiIR
+ EIR
+ AbsIR
+ AcosIR
+ AsinIR
+ AsinhIR
+ AtanIR
+ CbrtIR
+ CeilIR
+ CosIR
+ CoshIR
+ ErfIR
+ ExpIR
+ Exp2IR
+ FabsIR
+ FloorIR
+ LgammaIR
+ RintIR
+ SinIR
+ SinhIR
+ SqrtIR
+ TanIR
+ TanhIR
+ TgammaIR
+ TruncIR
+ IsFiniteIR
+ IsInfIR
+ IsNaNIR
+ Atan2IR
+ FmodIR
+ HypotIR
+ RemainderIR
+ PowIR
+ LdexpIR
+ LogIR
+ Log2IR
+ Log10IR
+ SigmoidIR
+ FuzzyEqualsIR
+ RoundIR
ModIR
RemIR
-)
-
+)
+
PEERDIR(
ydb/library/yql/udfs/common/math/lib
)
diff --git a/ydb/library/yql/udfs/common/string/string_udf.cpp b/ydb/library/yql/udfs/common/string/string_udf.cpp
index c468215090..b472ad4eae 100644
--- a/ydb/library/yql/udfs/common/string/string_udf.cpp
+++ b/ydb/library/yql/udfs/common/string/string_udf.cpp
@@ -72,16 +72,16 @@ namespace {
} \
}
-#define STROKA_ASCII_CASE_UDF(udfName, function) \
+#define STROKA_ASCII_CASE_UDF(udfName, function) \
SIMPLE_UDF(T##udfName, char*(TAutoMap<char*>)) { \
- TString input(args[0].AsStringRef()); \
- if (input.function()) { \
- return valueBuilder->NewString(input); \
- } else { \
- return args[0]; \
- } \
- }
-
+ TString input(args[0].AsStringRef()); \
+ if (input.function()) { \
+ return valueBuilder->NewString(input); \
+ } else { \
+ return args[0]; \
+ } \
+ }
+
#define STROKA_FIND_UDF(udfName, function) \
SIMPLE_UDF(T##udfName, bool(TOptional<char*>, char*)) { \
Y_UNUSED(valueBuilder); \
@@ -147,11 +147,11 @@ namespace {
XX(ToUpper, ToUpper) \
XX(ToTitle, ToTitle)
-#define STROKA_ASCII_CASE_UDF_MAP(XX) \
- XX(AsciiToLower, to_lower) \
- XX(AsciiToUpper, to_upper) \
- XX(AsciiToTitle, to_title)
-
+#define STROKA_ASCII_CASE_UDF_MAP(XX) \
+ XX(AsciiToLower, to_lower) \
+ XX(AsciiToUpper, to_upper) \
+ XX(AsciiToTitle, to_title)
+
#define STROKA_FIND_UDF_MAP(XX) \
XX(Contains, Contains) \
XX(StartsWith, StartsWith) \
@@ -197,10 +197,10 @@ namespace {
TStringBuf what(args[1].AsStringRef());
TStringBuf with(args[2].AsStringRef());
if (what.size() != 1) {
- UdfTerminate("Only one char is supported as second argument");
+ UdfTerminate("Only one char is supported as second argument");
}
if (with.size() != 1) {
- UdfTerminate("Only one char is supported as third argument");
+ UdfTerminate("Only one char is supported as third argument");
}
if (const auto index = result.find(what[0]); index != TStringBuf::npos) {
result.replace(index, 1, with.data());
@@ -214,10 +214,10 @@ namespace {
TStringBuf what(args[1].AsStringRef());
TStringBuf with(args[2].AsStringRef());
if (what.size() != 1) {
- UdfTerminate("Only one char is supported as second argument");
+ UdfTerminate("Only one char is supported as second argument");
}
if (with.size() != 1) {
- UdfTerminate("Only one char is supported as third argument");
+ UdfTerminate("Only one char is supported as third argument");
}
if (const auto index = result.rfind(what[0]); index != TStringBuf::npos) {
result.replace(index, 1, with.data());
@@ -239,7 +239,7 @@ namespace {
TString result(args[0].AsStringRef());
TStringBuf remove(args[1].AsStringRef());
if (remove.size() != 1) {
- UdfTerminate("Only one char is supported as second argument");
+ UdfTerminate("Only one char is supported as second argument");
}
if (const auto index = result.find(remove[0]); index != TStringBuf::npos) {
result.remove(index, 1);
@@ -252,7 +252,7 @@ namespace {
TString result(args[0].AsStringRef());
TStringBuf remove(args[1].AsStringRef());
if (remove.size() != 1) {
- UdfTerminate("Only one char is supported as second argument");
+ UdfTerminate("Only one char is supported as second argument");
}
if (const auto index = result.rfind(remove[0]); index != TStringBuf::npos) {
result.remove(index, 1);
@@ -512,46 +512,46 @@ namespace {
STRING_UNSAFE_UDF_MAP(STRING_UNSAFE_UDF)
STROKA_UDF_MAP(STROKA_UDF)
STROKA_CASE_UDF_MAP(STROKA_CASE_UDF)
- STROKA_ASCII_CASE_UDF_MAP(STROKA_ASCII_CASE_UDF)
+ STROKA_ASCII_CASE_UDF_MAP(STROKA_ASCII_CASE_UDF)
STROKA_FIND_UDF_MAP(STROKA_FIND_UDF)
STRING_TWO_ARGS_UDF_MAP(STRING_TWO_ARGS_UDF)
IS_ASCII_UDF_MAP(IS_ASCII_UDF)
SIMPLE_MODULE(TStringModule,
- STRING_UDF_MAP(STRING_REGISTER_UDF)
- STRING_UNSAFE_UDF_MAP(STRING_REGISTER_UDF)
- STROKA_UDF_MAP(STRING_REGISTER_UDF)
- STROKA_CASE_UDF_MAP(STRING_REGISTER_UDF)
- STROKA_ASCII_CASE_UDF_MAP(STRING_REGISTER_UDF)
- STROKA_FIND_UDF_MAP(STRING_REGISTER_UDF)
- STRING_TWO_ARGS_UDF_MAP(STRING_REGISTER_UDF)
- IS_ASCII_UDF_MAP(STRING_REGISTER_UDF)
- TCollapseText,
- TReplaceAll,
- TReplaceFirst,
- TReplaceLast,
- TRemoveAll,
- TRemoveFirst,
- TRemoveLast,
- TContains,
- TFind,
- TReverseFind,
- TSubstring,
- TSplitToList,
- TJoinFromList,
- TLevensteinDistance,
- TRightPad,
+ STRING_UDF_MAP(STRING_REGISTER_UDF)
+ STRING_UNSAFE_UDF_MAP(STRING_REGISTER_UDF)
+ STROKA_UDF_MAP(STRING_REGISTER_UDF)
+ STROKA_CASE_UDF_MAP(STRING_REGISTER_UDF)
+ STROKA_ASCII_CASE_UDF_MAP(STRING_REGISTER_UDF)
+ STROKA_FIND_UDF_MAP(STRING_REGISTER_UDF)
+ STRING_TWO_ARGS_UDF_MAP(STRING_REGISTER_UDF)
+ IS_ASCII_UDF_MAP(STRING_REGISTER_UDF)
+ TCollapseText,
+ TReplaceAll,
+ TReplaceFirst,
+ TReplaceLast,
+ TRemoveAll,
+ TRemoveFirst,
+ TRemoveLast,
+ TContains,
+ TFind,
+ TReverseFind,
+ TSubstring,
+ TSplitToList,
+ TJoinFromList,
+ TLevensteinDistance,
+ TRightPad,
TLeftPad,
- THex,
- TSHex,
- TBin,
- TSBin,
- THexText,
- TBinText,
- THumanReadableDuration,
- THumanReadableQuantity,
- THumanReadableBytes,
- TPrec,
+ THex,
+ TSHex,
+ TBin,
+ TSBin,
+ THexText,
+ TBinText,
+ THumanReadableDuration,
+ THumanReadableQuantity,
+ THumanReadableBytes,
+ TPrec,
TToByteList,
TFromByteList)
}
diff --git a/ydb/library/yql/utils/backtrace/backtrace.cpp b/ydb/library/yql/utils/backtrace/backtrace.cpp
index f9487f6671..fb39b13767 100644
--- a/ydb/library/yql/utils/backtrace/backtrace.cpp
+++ b/ydb/library/yql/utils/backtrace/backtrace.cpp
@@ -1,4 +1,4 @@
-#include "backtrace.h"
+#include "backtrace.h"
#include <llvm/DebugInfo/Symbolize/Symbolize.h>
#include <llvm/DebugInfo/Symbolize/DIPrinter.h>
@@ -14,30 +14,30 @@
#include <util/system/backtrace.h>
#include <util/system/type_name.h>
#include <util/system/execpath.h>
-#include <util/system/platform.h>
-#include <util/system/mlock.h>
+#include <util/system/platform.h>
+#include <util/system/mlock.h>
#ifdef _linux_
#include <dlfcn.h>
#include <link.h>
#endif
-#ifndef _win_
+#ifndef _win_
namespace {
-bool SetSignalHandler(int signo, void (*handler)(int)) {
- struct sigaction sa;
- memset(&sa, 0, sizeof(sa));
- sa.sa_flags = SA_RESETHAND;
- sa.sa_handler = handler;
- sigfillset(&sa.sa_mask);
- return sigaction(signo, &sa, nullptr) != -1;
-}
+bool SetSignalHandler(int signo, void (*handler)(int)) {
+ struct sigaction sa;
+ memset(&sa, 0, sizeof(sa));
+ sa.sa_flags = SA_RESETHAND;
+ sa.sa_handler = handler;
+ sigfillset(&sa.sa_mask);
+ return sigaction(signo, &sa, nullptr) != -1;
+}
} // namespace
-#endif // _win_
+#endif // _win_
void KikimrBackTrace() {
KikimrBacktraceFormatImpl(&Cerr);
@@ -62,37 +62,37 @@ int DlIterCallback(struct dl_phdr_info *info, size_t size, void *data)
}
#endif
-TAtomic BacktraceStarted = 0;
-
-class TRawOStreamProxy: public llvm::raw_ostream {
-public:
- TRawOStreamProxy(IOutputStream& out)
- : llvm::raw_ostream(true) // unbuffered
- , Slave_(out)
- {
- }
- void write_impl(const char* ptr, size_t size) override {
- Slave_.Write(ptr, size);
- }
- uint64_t current_pos() const override {
- return 0;
- }
- size_t preferred_buffer_size() const override {
- return 0;
- }
-private:
- IOutputStream& Slave_;
-};
-
-
+TAtomic BacktraceStarted = 0;
+
+class TRawOStreamProxy: public llvm::raw_ostream {
+public:
+ TRawOStreamProxy(IOutputStream& out)
+ : llvm::raw_ostream(true) // unbuffered
+ , Slave_(out)
+ {
+ }
+ void write_impl(const char* ptr, size_t size) override {
+ Slave_.Write(ptr, size);
+ }
+ uint64_t current_pos() const override {
+ return 0;
+ }
+ size_t preferred_buffer_size() const override {
+ return 0;
+ }
+private:
+ IOutputStream& Slave_;
+};
+
+
void KikimrBacktraceFormatImpl(IOutputStream* out) {
- if (out == &Cerr && !AtomicTryLock(&BacktraceStarted)) {
+ if (out == &Cerr && !AtomicTryLock(&BacktraceStarted)) {
return;
}
- // Unlock memory to avoid extra RSS consumption while touching debug info
- UnlockAllMemory();
-
+ // Unlock memory to avoid extra RSS consumption while touching debug info
+ UnlockAllMemory();
+
void* array[300];
const size_t s = BackTrace(array, Y_ARRAY_SIZE(array));
KikimrBacktraceFormatImpl(out, array, s);
@@ -108,7 +108,7 @@ void KikimrBacktraceFormatImpl(IOutputStream* out, void* const* stack, size_t st
dl_iterate_phdr(DlIterCallback, &dlls);
#endif
auto binaryPath = GetPersistentExecPath();
- DIPrinter printer(outStream, true, true, false);
+ DIPrinter printer(outStream, true, true, false);
LLVMSymbolizer::Options opts;
LLVMSymbolizer symbolyzer(opts);
for (const auto i : xrange(stackSize)) {
@@ -139,13 +139,13 @@ void KikimrBacktraceFormatImpl(IOutputStream* out, void* const* stack, size_t st
printer << value;
} else {
- logAllUnhandledErrors(resOrErr.takeError(), outStream,
+ logAllUnhandledErrors(resOrErr.takeError(), outStream,
"LLVMSymbolizer: error reading file: ");
}
}
}
-void PrintBacktraceToStderr(int signum)
+void PrintBacktraceToStderr(int signum)
{
if (!NMalloc::IsAllocatorCorrupted) {
/* we want memory allocation for backtrace printing */
@@ -160,17 +160,17 @@ void PrintBacktraceToStderr(int signum)
void EnableKikimrBacktraceFormat()
{
- SetFormatBackTraceFn(KikimrBacktraceFormatImpl);
+ SetFormatBackTraceFn(KikimrBacktraceFormatImpl);
}
-void SetFatalSignalHandler(void (*handler)(int))
+void SetFatalSignalHandler(void (*handler)(int))
{
- Y_UNUSED(handler);
+ Y_UNUSED(handler);
#ifndef _win_
- for (int signo: {SIGSEGV, SIGILL, SIGABRT, SIGFPE}) {
- if (!SetSignalHandler(signo, handler)) {
- ythrow TSystemError() << "Cannot set handler for signal " << strsignal(signo);
- }
+ for (int signo: {SIGSEGV, SIGILL, SIGABRT, SIGFPE}) {
+ if (!SetSignalHandler(signo, handler)) {
+ ythrow TSystemError() << "Cannot set handler for signal " << strsignal(signo);
+ }
}
#endif
}
diff --git a/ydb/library/yql/utils/backtrace/backtrace.h b/ydb/library/yql/utils/backtrace/backtrace.h
index 487547e63b..24d03c085d 100644
--- a/ydb/library/yql/utils/backtrace/backtrace.h
+++ b/ydb/library/yql/utils/backtrace/backtrace.h
@@ -8,6 +8,6 @@ void KikimrBacktraceFormatImpl(IOutputStream* out);
void KikimrBacktraceFormatImpl(IOutputStream* out, void* const* stack, size_t stackSize);
void KikimrBackTrace();
void EnableKikimrBacktraceFormat();
-void PrintBacktraceToStderr(int signum);
-void SetFatalSignalHandler(void (*handler)(int));
+void PrintBacktraceToStderr(int signum);
+void SetFatalSignalHandler(void (*handler)(int));
diff --git a/ydb/library/yql/utils/debug_info.cpp b/ydb/library/yql/utils/debug_info.cpp
index ff56166695..20e6b0e34e 100644
--- a/ydb/library/yql/utils/debug_info.cpp
+++ b/ydb/library/yql/utils/debug_info.cpp
@@ -2,10 +2,10 @@
#include <util/system/thread.h>
#include <util/system/tls.h>
-#include <util/stream/file.h>
-#include <util/generic/string.h>
+#include <util/stream/file.h>
+#include <util/generic/string.h>
-#include <string.h>
+#include <string.h>
namespace NYql {
@@ -37,14 +37,14 @@ void SetCurrentOperationId(const char* operationId) {
}
long GetRunnigThreadsCount() {
- TString procStat = TFileInput("/proc/self/stat").ReadAll();
- long num_threads = -2; // Number of threads in this process (since Linux 2.6)
+ TString procStat = TFileInput("/proc/self/stat").ReadAll();
+ long num_threads = -2; // Number of threads in this process (since Linux 2.6)
int n = sscanf(procStat.data(),
- "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %*u %*u %*d %*d %*d %*d %ld",
- &num_threads);
+ "%*d %*s %*c %*d %*d %*d %*d %*d %*u %*u %*u %*u %*u %*u %*u %*d %*d %*d %*d %ld",
+ &num_threads);
- return n == 1 ? num_threads : -2;
+ return n == 1 ? num_threads : -2;
}
} // namespace NYql
diff --git a/ydb/library/yql/utils/failure_injector/failure_injector.cpp b/ydb/library/yql/utils/failure_injector/failure_injector.cpp
index db368a0aa6..82c17c0495 100644
--- a/ydb/library/yql/utils/failure_injector/failure_injector.cpp
+++ b/ydb/library/yql/utils/failure_injector/failure_injector.cpp
@@ -1,52 +1,52 @@
#include "failure_injector.h"
-
+
#include <ydb/library/yql/utils/log/log.h>
#include <ydb/library/yql/utils/yql_panic.h>
-#include <util/generic/singleton.h>
-
-namespace NYql {
-
-void TFailureInjector::Activate() {
- Singleton<TFailureInjector>()->ActivateImpl();
-}
-
-void TFailureInjector::Set(std::string_view name, ui64 skip, ui64 countOfFails) {
- Singleton<TFailureInjector>()->SetImpl(name, skip, countOfFails);
-}
-
-void TFailureInjector::Reach(std::string_view name, std::function<void()> action) {
- Singleton<TFailureInjector>()->ReachImpl(name, action);
-}
-
-void TFailureInjector::ActivateImpl() {
- Enabled_.store(true);
- YQL_LOG(DEBUG) << "TFailureInjector::Activate";
-}
-
-void TFailureInjector::ReachImpl(std::string_view name, std::function<void()> action) {
- if (!Enabled_.load()) {
- return;
- }
- with_lock(Lock) {
- if (auto failureSpec = FailureSpecs.FindPtr(name)) {
- YQL_LOG(DEBUG) << "TFailureInjector::Reach: " << name << ", Skip=" << failureSpec->Skip << ", Fails=" << failureSpec->CountOfFails;
- if (failureSpec->Skip > 0) {
- --failureSpec->Skip;
- } else if (failureSpec->CountOfFails > 0) {
- YQL_LOG(DEBUG) << "TFailureInjector::OnReach: " << name;
- --failureSpec->CountOfFails;
- action();
+#include <util/generic/singleton.h>
+
+namespace NYql {
+
+void TFailureInjector::Activate() {
+ Singleton<TFailureInjector>()->ActivateImpl();
+}
+
+void TFailureInjector::Set(std::string_view name, ui64 skip, ui64 countOfFails) {
+ Singleton<TFailureInjector>()->SetImpl(name, skip, countOfFails);
+}
+
+void TFailureInjector::Reach(std::string_view name, std::function<void()> action) {
+ Singleton<TFailureInjector>()->ReachImpl(name, action);
+}
+
+void TFailureInjector::ActivateImpl() {
+ Enabled_.store(true);
+ YQL_LOG(DEBUG) << "TFailureInjector::Activate";
+}
+
+void TFailureInjector::ReachImpl(std::string_view name, std::function<void()> action) {
+ if (!Enabled_.load()) {
+ return;
+ }
+ with_lock(Lock) {
+ if (auto failureSpec = FailureSpecs.FindPtr(name)) {
+ YQL_LOG(DEBUG) << "TFailureInjector::Reach: " << name << ", Skip=" << failureSpec->Skip << ", Fails=" << failureSpec->CountOfFails;
+ if (failureSpec->Skip > 0) {
+ --failureSpec->Skip;
+ } else if (failureSpec->CountOfFails > 0) {
+ YQL_LOG(DEBUG) << "TFailureInjector::OnReach: " << name;
+ --failureSpec->CountOfFails;
+ action();
}
}
}
-}
+}
-void TFailureInjector::SetImpl(std::string_view name, ui64 skip, ui64 countOfFails) {
- with_lock(Lock) {
- YQL_ENSURE(countOfFails > 0, "failure " << name << ", 'countOfFails' must be positive");
- FailureSpecs[TString{name}] = TFailureSpec{skip, countOfFails};
+void TFailureInjector::SetImpl(std::string_view name, ui64 skip, ui64 countOfFails) {
+ with_lock(Lock) {
+ YQL_ENSURE(countOfFails > 0, "failure " << name << ", 'countOfFails' must be positive");
+ FailureSpecs[TString{name}] = TFailureSpec{skip, countOfFails};
}
-}
+}
-} // NYql
+} // NYql
diff --git a/ydb/library/yql/utils/failure_injector/failure_injector.h b/ydb/library/yql/utils/failure_injector/failure_injector.h
index ea2304f5d8..81f9e6c713 100644
--- a/ydb/library/yql/utils/failure_injector/failure_injector.h
+++ b/ydb/library/yql/utils/failure_injector/failure_injector.h
@@ -3,32 +3,32 @@
#include <util/generic/hash.h>
#include <util/generic/string.h>
#include <util/system/mutex.h>
-
-#include <string_view>
+
+#include <string_view>
namespace NYql {
-class TFailureInjector {
-public:
- static void Activate();
+class TFailureInjector {
+public:
+ static void Activate();
- static void Set(std::string_view name, ui64 skip, ui64 countOfFails);
- static void Reach(std::string_view name, std::function<void()> action);
+ static void Set(std::string_view name, ui64 skip, ui64 countOfFails);
+ static void Reach(std::string_view name, std::function<void()> action);
-private:
- void ActivateImpl();
+private:
+ void ActivateImpl();
- void SetImpl(std::string_view name, ui64 skip, ui64 countOfFails);
- void ReachImpl(std::string_view name, std::function<void()> action);
+ void SetImpl(std::string_view name, ui64 skip, ui64 countOfFails);
+ void ReachImpl(std::string_view name, std::function<void()> action);
- struct TFailureSpec {
- ui64 Skip;
- ui64 CountOfFails;
- };
+ struct TFailureSpec {
+ ui64 Skip;
+ ui64 CountOfFails;
+ };
- std::atomic<bool> Enabled_ = false;
- THashMap<TString, TFailureSpec> FailureSpecs;
- TMutex Lock;
-};
+ std::atomic<bool> Enabled_ = false;
+ THashMap<TString, TFailureSpec> FailureSpecs;
+ TMutex Lock;
+};
-} // NYql
+} // NYql
diff --git a/ydb/library/yql/utils/failure_injector/failure_injector_ut.cpp b/ydb/library/yql/utils/failure_injector/failure_injector_ut.cpp
index dd59d9283d..63fa30c1e1 100644
--- a/ydb/library/yql/utils/failure_injector/failure_injector_ut.cpp
+++ b/ydb/library/yql/utils/failure_injector/failure_injector_ut.cpp
@@ -1,11 +1,11 @@
-#include "failure_injector.h"
+#include "failure_injector.h"
#include <ydb/library/yql/utils/log/log.h>
-
-#include <library/cpp/testing/unittest/registar.h>
-
-#include <util/datetime/base.h>
-
+
+#include <library/cpp/testing/unittest/registar.h>
+
+#include <util/datetime/base.h>
+
#include <chrono>
using namespace NYql;
@@ -26,62 +26,62 @@ void SetUpLogger() {
}
Y_UNIT_TEST_SUITE(TFailureInjectorTests) {
- Y_UNIT_TEST(BasicFailureTest) {
- SetUpLogger();
- std::atomic<bool> called;
- called.store(false);
- auto behavior = [&called] { OnReach(called); };
- TFailureInjector::Reach("misc_failure", behavior);
- UNIT_ASSERT_EQUAL(false, called.load());
- TFailureInjector::Activate();
- TFailureInjector::Set("misc_failure", 0, 1);
- TFailureInjector::Reach("misc_failure", behavior);
- UNIT_ASSERT_EQUAL(true, called.load());
- }
+ Y_UNIT_TEST(BasicFailureTest) {
+ SetUpLogger();
+ std::atomic<bool> called;
+ called.store(false);
+ auto behavior = [&called] { OnReach(called); };
+ TFailureInjector::Reach("misc_failure", behavior);
+ UNIT_ASSERT_EQUAL(false, called.load());
+ TFailureInjector::Activate();
+ TFailureInjector::Set("misc_failure", 0, 1);
+ TFailureInjector::Reach("misc_failure", behavior);
+ UNIT_ASSERT_EQUAL(true, called.load());
+ }
- Y_UNIT_TEST(CheckSkipTest) {
- SetUpLogger();
- std::atomic<bool> called;
- called.store(false);
- auto behavior = [&called] { OnReach(called); };
- TFailureInjector::Activate();
- TFailureInjector::Set("misc_failure", 1, 1);
+ Y_UNIT_TEST(CheckSkipTest) {
+ SetUpLogger();
+ std::atomic<bool> called;
+ called.store(false);
+ auto behavior = [&called] { OnReach(called); };
+ TFailureInjector::Activate();
+ TFailureInjector::Set("misc_failure", 1, 1);
- TFailureInjector::Reach("misc_failure", behavior);
- UNIT_ASSERT_EQUAL(false, called.load());
- TFailureInjector::Reach("misc_failure", behavior);
- UNIT_ASSERT_EQUAL(true, called.load());
- }
+ TFailureInjector::Reach("misc_failure", behavior);
+ UNIT_ASSERT_EQUAL(false, called.load());
+ TFailureInjector::Reach("misc_failure", behavior);
+ UNIT_ASSERT_EQUAL(true, called.load());
+ }
- Y_UNIT_TEST(CheckFailCountTest) {
- SetUpLogger();
- int called = 0;
- auto behavior = [&called] { ++called; };
- TFailureInjector::Activate();
- TFailureInjector::Set("misc_failure", 1, 2);
+ Y_UNIT_TEST(CheckFailCountTest) {
+ SetUpLogger();
+ int called = 0;
+ auto behavior = [&called] { ++called; };
+ TFailureInjector::Activate();
+ TFailureInjector::Set("misc_failure", 1, 2);
+
+ TFailureInjector::Reach("misc_failure", behavior);
+ UNIT_ASSERT_EQUAL(0, called);
+ TFailureInjector::Reach("misc_failure", behavior);
+ UNIT_ASSERT_EQUAL(1, called);
+ TFailureInjector::Reach("misc_failure", behavior);
+ UNIT_ASSERT_EQUAL(2, called);
+ TFailureInjector::Reach("misc_failure", behavior);
+ UNIT_ASSERT_EQUAL(2, called);
+ TFailureInjector::Reach("misc_failure", behavior);
+ UNIT_ASSERT_EQUAL(2, called);
+ }
+
+ Y_UNIT_TEST(SlowDownTest) {
+ SetUpLogger();
+ TFailureInjector::Activate();
+ TFailureInjector::Set("misc_failure", 0, 1);
- TFailureInjector::Reach("misc_failure", behavior);
- UNIT_ASSERT_EQUAL(0, called);
- TFailureInjector::Reach("misc_failure", behavior);
- UNIT_ASSERT_EQUAL(1, called);
- TFailureInjector::Reach("misc_failure", behavior);
- UNIT_ASSERT_EQUAL(2, called);
- TFailureInjector::Reach("misc_failure", behavior);
- UNIT_ASSERT_EQUAL(2, called);
- TFailureInjector::Reach("misc_failure", behavior);
- UNIT_ASSERT_EQUAL(2, called);
- }
-
- Y_UNIT_TEST(SlowDownTest) {
- SetUpLogger();
- TFailureInjector::Activate();
- TFailureInjector::Set("misc_failure", 0, 1);
-
- auto start = system_clock::now();
- TFailureInjector::Reach("misc_failure", [] { ::Sleep(TDuration::Seconds(5)); });
- auto finish = system_clock::now();
- auto duration = duration_cast<std::chrono::seconds>(finish - start);
- YQL_LOG(DEBUG) << "Duration :" << duration.count();
- UNIT_ASSERT_GE(duration.count(), 5);
- }
-}
+ auto start = system_clock::now();
+ TFailureInjector::Reach("misc_failure", [] { ::Sleep(TDuration::Seconds(5)); });
+ auto finish = system_clock::now();
+ auto duration = duration_cast<std::chrono::seconds>(finish - start);
+ YQL_LOG(DEBUG) << "Duration :" << duration.count();
+ UNIT_ASSERT_GE(duration.count(), 5);
+ }
+}
diff --git a/ydb/library/yql/utils/fetch/fetch.cpp b/ydb/library/yql/utils/fetch/fetch.cpp
index 100c88c385..801efa329b 100644
--- a/ydb/library/yql/utils/fetch/fetch.cpp
+++ b/ydb/library/yql/utils/fetch/fetch.cpp
@@ -1,51 +1,51 @@
-#include "fetch.h"
-
+#include "fetch.h"
+
#include <ydb/library/yql/utils/log/log.h>
-
+
#include <library/cpp/openssl/io/stream.h>
#include <library/cpp/http/misc/httpcodes.h>
-#include <library/cpp/charset/ci_string.h>
-
-#include <util/network/socket.h>
-#include <util/string/cast.h>
-#include <util/generic/strbuf.h>
-
-namespace NYql {
-
-namespace {
-
-THttpURL ParseURL(const TStringBuf addr, NUri::TParseFlags features) {
- THttpURL url;
+#include <library/cpp/charset/ci_string.h>
+
+#include <util/network/socket.h>
+#include <util/string/cast.h>
+#include <util/generic/strbuf.h>
+
+namespace NYql {
+
+namespace {
+
+THttpURL ParseURL(const TStringBuf addr, NUri::TParseFlags features) {
+ THttpURL url;
THttpURL::TParsedState parsedState = url.Parse(addr, features, nullptr, 65536);
- if (THttpURL::ParsedOK != parsedState) {
- ythrow yexception() << "Bad URL: \"" << addr << "\", " << HttpURLParsedStateToString(parsedState);
- }
- return url;
-}
-
-class TFetchResultImpl: public IFetchResult {
-public:
- TFetchResultImpl(const THttpURL& url, const THttpHeaders& additionalHeaders, TDuration timeout) {
+ if (THttpURL::ParsedOK != parsedState) {
+ ythrow yexception() << "Bad URL: \"" << addr << "\", " << HttpURLParsedStateToString(parsedState);
+ }
+ return url;
+}
+
+class TFetchResultImpl: public IFetchResult {
+public:
+ TFetchResultImpl(const THttpURL& url, const THttpHeaders& additionalHeaders, TDuration timeout) {
TString host = url.Get(THttpURL::FieldHost);
TString path = url.PrintS(THttpURL::FlagPath | THttpURL::FlagQuery);
- const char* p = url.Get(THttpURL::FieldPort);
- ui16 port = 80;
- bool https = false;
-
+ const char* p = url.Get(THttpURL::FieldPort);
+ ui16 port = 80;
+ bool https = false;
+
if (url.Get(THttpURL::FieldScheme) == TStringBuf("https")) {
- port = 443;
- https = true;
- }
-
- if (p) {
- port = FromString<ui16>(p);
- }
-
+ port = 443;
+ https = true;
+ }
+
+ if (p) {
+ port = FromString<ui16>(p);
+ }
+
TString req;
- {
- TStringOutput rqs(req);
+ {
+ TStringOutput rqs(req);
TStringBuf userAgent = "User-Agent: Mozilla/5.0 (compatible; YQL/1.0)";
-
+
IOutputStream::TPart request[] = {
IOutputStream::TPart("GET ", 4),
IOutputStream::TPart(path.data(), path.size()),
@@ -56,106 +56,106 @@ public:
IOutputStream::TPart::CrLf(),
IOutputStream::TPart(userAgent.data(), userAgent.size()),
IOutputStream::TPart::CrLf(),
- };
+ };
rqs.Write(request, Y_ARRAY_SIZE(request));
- if (!additionalHeaders.Empty()) {
- additionalHeaders.OutTo(&rqs);
- }
- rqs << "\r\n";
- }
-
- Socket.Reset(new TSocket(TNetworkAddress(host, port), timeout));
- SocketInput.Reset(new TSocketInput(*Socket));
- SocketOutput.Reset(new TSocketOutput(*Socket));
+ if (!additionalHeaders.Empty()) {
+ additionalHeaders.OutTo(&rqs);
+ }
+ rqs << "\r\n";
+ }
+
+ Socket.Reset(new TSocket(TNetworkAddress(host, port), timeout));
+ SocketInput.Reset(new TSocketInput(*Socket));
+ SocketOutput.Reset(new TSocketOutput(*Socket));
Socket->SetSocketTimeout(timeout.Seconds(), timeout.MilliSeconds() % 1000);
- if (https) {
- Ssl.Reset(new TOpenSslClientIO(SocketInput.Get(), SocketOutput.Get()));
- }
-
- {
+ if (https) {
+ Ssl.Reset(new TOpenSslClientIO(SocketInput.Get(), SocketOutput.Get()));
+ }
+
+ {
THttpOutput ho(Ssl ? (IOutputStream*)Ssl.Get() : (IOutputStream*)SocketOutput.Get());
- (ho << req).Finish();
- }
+ (ho << req).Finish();
+ }
HttpInput.Reset(new THttpInput(Ssl ? (IInputStream*)Ssl.Get() : (IInputStream*)SocketInput.Get()));
- }
-
+ }
+
THttpInput& GetStream() override {
- return *HttpInput;
- }
-
+ return *HttpInput;
+ }
+
unsigned GetRetCode() override {
- return ParseHttpRetCode(HttpInput->FirstLine());
- }
-
+ return ParseHttpRetCode(HttpInput->FirstLine());
+ }
+
THttpURL GetRedirectURL(const THttpURL& baseUrl) override {
- for (auto i = HttpInput->Headers().Begin(); i != HttpInput->Headers().End(); ++i) {
+ for (auto i = HttpInput->Headers().Begin(); i != HttpInput->Headers().End(); ++i) {
if (0 == TCiString::compare(i->Name(), TStringBuf("location"))) {
- THttpURL target = ParseURL(i->Value(), THttpURL::FeaturesAll | NUri::TFeature::FeatureConvertHostIDN);
- if (!target.IsValidAbs()) {
- target.Merge(baseUrl);
- }
- return target;
- }
- }
- ythrow yexception() << "Unknown redirect location from " << baseUrl.PrintS();
- }
-
- static TFetchResultPtr Fetch(const THttpURL& url, const THttpHeaders& additionalHeaders, const TDuration& timeout) {
- return new TFetchResultImpl(url, additionalHeaders, timeout);
- }
-
-private:
- THolder<TSocket> Socket;
- THolder<TSocketInput> SocketInput;
- THolder<TSocketOutput> SocketOutput;
- THolder<TOpenSslClientIO> Ssl;
- THolder<THttpInput> HttpInput;
-};
-
-inline bool IsRedirectCode(unsigned code) {
- switch (code) {
- case HTTP_MOVED_PERMANENTLY:
- case HTTP_FOUND:
- case HTTP_SEE_OTHER:
- case HTTP_TEMPORARY_REDIRECT:
- return true;
- }
- return false;
-}
-
-inline bool IsRetryCode(unsigned code) {
- switch (code) {
- case HTTP_REQUEST_TIME_OUT:
- case HTTP_AUTHENTICATION_TIMEOUT:
- case HTTP_TOO_MANY_REQUESTS:
- case HTTP_GATEWAY_TIME_OUT:
- case HTTP_SERVICE_UNAVAILABLE:
- return true;
- }
- return false;
-}
-
-} // unnamed
-
-THttpURL ParseURL(const TStringBuf addr) {
- return ParseURL(addr, THttpURL::FeaturesAll | NUri::TFeature::FeatureConvertHostIDN | NUri::TFeature::FeatureNoRelPath);
-}
-
-TFetchResultPtr Fetch(const THttpURL& url, const THttpHeaders& additionalHeaders, const TDuration& timeout, size_t retries, size_t redirects) {
- THttpURL currentUrl = url;
- for (size_t fetchNum = 0; fetchNum < redirects; ++fetchNum) {
- unsigned responseCode = 0;
- TFetchResultPtr fr;
- size_t fetchTry = 0;
- do {
- fr = TFetchResultImpl::Fetch(currentUrl, additionalHeaders, timeout);
- responseCode = fr->GetRetCode();
- } while (IsRetryCode(responseCode) && ++fetchTry < retries);
-
- if (responseCode >= 200 && responseCode < 300) {
- return fr;
+ THttpURL target = ParseURL(i->Value(), THttpURL::FeaturesAll | NUri::TFeature::FeatureConvertHostIDN);
+ if (!target.IsValidAbs()) {
+ target.Merge(baseUrl);
+ }
+ return target;
+ }
+ }
+ ythrow yexception() << "Unknown redirect location from " << baseUrl.PrintS();
+ }
+
+ static TFetchResultPtr Fetch(const THttpURL& url, const THttpHeaders& additionalHeaders, const TDuration& timeout) {
+ return new TFetchResultImpl(url, additionalHeaders, timeout);
+ }
+
+private:
+ THolder<TSocket> Socket;
+ THolder<TSocketInput> SocketInput;
+ THolder<TSocketOutput> SocketOutput;
+ THolder<TOpenSslClientIO> Ssl;
+ THolder<THttpInput> HttpInput;
+};
+
+inline bool IsRedirectCode(unsigned code) {
+ switch (code) {
+ case HTTP_MOVED_PERMANENTLY:
+ case HTTP_FOUND:
+ case HTTP_SEE_OTHER:
+ case HTTP_TEMPORARY_REDIRECT:
+ return true;
+ }
+ return false;
+}
+
+inline bool IsRetryCode(unsigned code) {
+ switch (code) {
+ case HTTP_REQUEST_TIME_OUT:
+ case HTTP_AUTHENTICATION_TIMEOUT:
+ case HTTP_TOO_MANY_REQUESTS:
+ case HTTP_GATEWAY_TIME_OUT:
+ case HTTP_SERVICE_UNAVAILABLE:
+ return true;
+ }
+ return false;
+}
+
+} // unnamed
+
+THttpURL ParseURL(const TStringBuf addr) {
+ return ParseURL(addr, THttpURL::FeaturesAll | NUri::TFeature::FeatureConvertHostIDN | NUri::TFeature::FeatureNoRelPath);
+}
+
+TFetchResultPtr Fetch(const THttpURL& url, const THttpHeaders& additionalHeaders, const TDuration& timeout, size_t retries, size_t redirects) {
+ THttpURL currentUrl = url;
+ for (size_t fetchNum = 0; fetchNum < redirects; ++fetchNum) {
+ unsigned responseCode = 0;
+ TFetchResultPtr fr;
+ size_t fetchTry = 0;
+ do {
+ fr = TFetchResultImpl::Fetch(currentUrl, additionalHeaders, timeout);
+ responseCode = fr->GetRetCode();
+ } while (IsRetryCode(responseCode) && ++fetchTry < retries);
+
+ if (responseCode >= 200 && responseCode < 300) {
+ return fr;
}
if (responseCode == HTTP_NOT_MODIFIED) {
@@ -163,20 +163,20 @@ TFetchResultPtr Fetch(const THttpURL& url, const THttpHeaders& additionalHeaders
}
if (IsRedirectCode(responseCode)) {
- currentUrl = fr->GetRedirectURL(currentUrl);
- YQL_LOG(INFO) << "Got redirect to " << currentUrl.PrintS();
+ currentUrl = fr->GetRedirectURL(currentUrl);
+ YQL_LOG(INFO) << "Got redirect to " << currentUrl.PrintS();
continue;
- }
-
- TString errorBody;
- try {
- errorBody = fr->GetStream().ReadAll();
- } catch (...) {
- }
-
- ythrow yexception() << "Failed to fetch url '" << currentUrl.PrintS() << "' with code " << responseCode << ", body: " << errorBody;
- }
- ythrow yexception() << "Failed to fetch url '" << currentUrl.PrintS() << "': too many redirects";
-}
-
-} // NYql
+ }
+
+ TString errorBody;
+ try {
+ errorBody = fr->GetStream().ReadAll();
+ } catch (...) {
+ }
+
+ ythrow yexception() << "Failed to fetch url '" << currentUrl.PrintS() << "' with code " << responseCode << ", body: " << errorBody;
+ }
+ ythrow yexception() << "Failed to fetch url '" << currentUrl.PrintS() << "': too many redirects";
+}
+
+} // NYql
diff --git a/ydb/library/yql/utils/fetch/fetch.h b/ydb/library/yql/utils/fetch/fetch.h
index d9e1c3c1a5..a808882380 100644
--- a/ydb/library/yql/utils/fetch/fetch.h
+++ b/ydb/library/yql/utils/fetch/fetch.h
@@ -1,24 +1,24 @@
-#pragma once
-
+#pragma once
+
#include <library/cpp/uri/http_url.h>
#include <library/cpp/http/io/headers.h>
#include <library/cpp/http/io/stream.h>
-
-#include <util/datetime/base.h>
+
+#include <util/datetime/base.h>
#include <util/generic/string.h>
-#include <util/generic/ptr.h>
-
-namespace NYql {
-
-struct IFetchResult: public TThrRefBase {
- virtual THttpInput& GetStream() = 0;
- virtual unsigned GetRetCode() = 0;
- virtual THttpURL GetRedirectURL(const THttpURL& baseUrl) = 0;
-};
-
-using TFetchResultPtr = TIntrusivePtr<IFetchResult>;
-
-THttpURL ParseURL(const TStringBuf addr);
-TFetchResultPtr Fetch(const THttpURL& url, const THttpHeaders& additionalHeaders = {}, const TDuration& timeout = TDuration::Max(), size_t retries = 3, size_t redirects = 10);
-
-} // NYql
+#include <util/generic/ptr.h>
+
+namespace NYql {
+
+struct IFetchResult: public TThrRefBase {
+ virtual THttpInput& GetStream() = 0;
+ virtual unsigned GetRetCode() = 0;
+ virtual THttpURL GetRedirectURL(const THttpURL& baseUrl) = 0;
+};
+
+using TFetchResultPtr = TIntrusivePtr<IFetchResult>;
+
+THttpURL ParseURL(const TStringBuf addr);
+TFetchResultPtr Fetch(const THttpURL& url, const THttpHeaders& additionalHeaders = {}, const TDuration& timeout = TDuration::Max(), size_t retries = 3, size_t redirects = 10);
+
+} // NYql
diff --git a/ydb/library/yql/utils/fetch/ya.make b/ydb/library/yql/utils/fetch/ya.make
index f36bfe247c..343afc8169 100644
--- a/ydb/library/yql/utils/fetch/ya.make
+++ b/ydb/library/yql/utils/fetch/ya.make
@@ -1,18 +1,18 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(g:yql)
-
-SRCS(
- fetch.cpp
-)
-
-PEERDIR(
+
+SRCS(
+ fetch.cpp
+)
+
+PEERDIR(
library/cpp/charset
library/cpp/http/io
library/cpp/http/misc
library/cpp/openssl/io
library/cpp/uri
ydb/library/yql/utils/log
-)
-
-END()
+)
+
+END()
diff --git a/ydb/library/yql/utils/log/context.cpp b/ydb/library/yql/utils/log/context.cpp
index b591c42a77..f9fb844f07 100644
--- a/ydb/library/yql/utils/log/context.cpp
+++ b/ydb/library/yql/utils/log/context.cpp
@@ -1,19 +1,19 @@
-#include "context.h"
+#include "context.h"
#include "log.h"
-
+
#include <util/thread/singleton.h>
+
-
-namespace NYql {
+namespace NYql {
namespace NLog {
namespace {
-
+
struct TThrowedLogContext {
TString LocationWithLogContext; // separated with ': '
};
-} // namspace
-
+} // namspace
+
void OutputLogCtx(IOutputStream* out, bool withBraces) {
const NImpl::TLogContextListItem* ctxList = NImpl::GetLogContextList();
@@ -26,8 +26,8 @@ void OutputLogCtx(IOutputStream* out, bool withBraces) {
NImpl::TLogContextListItem* ctxItem = ctxList->Next;
{ // special case for outputing the first element
- const TString* it = ctxItem->begin();
- const TString* end = ctxItem->end();
+ const TString* it = ctxItem->begin();
+ const TString* end = ctxItem->end();
(*out) << *it++;
for (; it != end; ++it) {
@@ -39,7 +39,7 @@ void OutputLogCtx(IOutputStream* out, bool withBraces) {
// output remaining elements
while (ctxItem != ctxList) {
- for (const TString& name: *ctxItem) {
+ for (const TString& name: *ctxItem) {
(*out) << '/' << name;
}
ctxItem = ctxItem->Next;
@@ -49,12 +49,12 @@ void OutputLogCtx(IOutputStream* out, bool withBraces) {
(*out) << TStringBuf("} ");
}
}
-}
-
+}
+
NImpl::TLogContextListItem* NImpl::GetLogContextList() {
return FastTlsSingleton<NImpl::TLogContextListItem>();
-}
-
+}
+
TString CurrentLogContextPath() {
TStringStream ss;
OutputLogCtx(&ss, false);
@@ -71,15 +71,15 @@ TAutoPtr<TLogElement> TContextPreprocessor::Preprocess(
{
OutputLogCtx(element.Get(), true);
return element;
-}
-
+}
+
void TYqlLogContextLocation::SetThrowedLogContextPath() const {
TStringStream ss;
ss << Location_ << TStringBuf(": ");
OutputLogCtx(&ss, true);
TThrowedLogContext* tlc = FastTlsSingleton<TThrowedLogContext>();
tlc->LocationWithLogContext = ss.Str();
-}
-
+}
+
} // namespace NLog
-} // namespace NYql
+} // namespace NYql
diff --git a/ydb/library/yql/utils/log/context.h b/ydb/library/yql/utils/log/context.h
index ab6e9430e5..48894a43e3 100644
--- a/ydb/library/yql/utils/log/context.h
+++ b/ydb/library/yql/utils/log/context.h
@@ -1,11 +1,11 @@
-#pragma once
-
-#include <util/system/defaults.h>
+#pragma once
+
+#include <util/system/defaults.h>
#include <util/generic/strbuf.h>
#include <util/stream/output.h>
#include <util/system/src_location.h>
#include <util/system/yassert.h>
-
+
// continues existing contexts chain
@@ -38,10 +38,10 @@
class TLogElement;
-namespace NYql {
+namespace NYql {
namespace NLog {
namespace NImpl {
-
+
/**
* @brief Represents item of logging context list.
*/
@@ -57,12 +57,12 @@ struct TLogContextListItem {
{
}
- const TString* begin() const {
+ const TString* begin() const {
auto* ptr = reinterpret_cast<const ui8*>(this);
- return reinterpret_cast<const TString*>(ptr + sizeof(*this));
+ return reinterpret_cast<const TString*>(ptr + sizeof(*this));
}
- const TString* end() const {
+ const TString* end() const {
return begin() + NamesCount;
}
@@ -99,11 +99,11 @@ TLogContextListItem* GetLogContextList();
*/
template <size_t Size>
class TLogContext: public NImpl::TLogContextListItem {
-public:
+public:
template <typename... TArgs>
TLogContext(TArgs... args)
: TLogContextListItem(Size)
- , Names_{{ TString{std::forward<TArgs>(args)}... }}
+ , Names_{{ TString{std::forward<TArgs>(args)}... }}
{
LinkBefore(NImpl::GetLogContextList());
}
@@ -112,14 +112,14 @@ public:
Unlink();
}
- explicit inline operator bool() const noexcept {
- return true;
- }
-
-private:
- std::array<TString, Size> Names_;
-};
+ explicit inline operator bool() const noexcept {
+ return true;
+ }
+private:
+ std::array<TString, Size> Names_;
+};
+
/**
* @brief Special Root context elements which replaces previous log context
* list head by itself and restores previous one on destruction.
@@ -130,7 +130,7 @@ public:
template <typename... TArgs>
TRootLogContext(TArgs... args)
: TLogContextListItem(Size)
- , Names_{{ TString{std::forward<TArgs>(args)}... }}
+ , Names_{{ TString{std::forward<TArgs>(args)}... }}
{
NImpl::TLogContextListItem* ctxList = NImpl::GetLogContextList();
PrevLogContextHead_.Prev = ctxList->Prev;
@@ -151,7 +151,7 @@ public:
}
private:
- std::array<TString, Size> Names_;
+ std::array<TString, Size> Names_;
NImpl::TLogContextListItem PrevLogContextHead_;
};
@@ -186,14 +186,14 @@ TString ThrowedLogContextPath();
*/
struct TContextPreprocessor {
static TAutoPtr<TLogElement> Preprocess(TAutoPtr<TLogElement> element);
-};
-
-/**
- * @brief Outputs current logger context into stream
- */
-void OutputLogCtx(IOutputStream* out, bool withBraces);
-
+};
+
/**
+ * @brief Outputs current logger context into stream
+ */
+void OutputLogCtx(IOutputStream* out, bool withBraces);
+
+/**
* @brief Outputs current logger context into exception message.
*/
class TYqlLogContextLocation {
@@ -202,9 +202,9 @@ public:
: Location_(location.File, location.Line)
{
}
-
+
void SetThrowedLogContextPath() const;
-
+
template <class T>
inline T&& operator+(T&& t) {
SetThrowedLogContextPath();
@@ -214,6 +214,6 @@ public:
private:
TSourceLocation Location_;
};
-
+
} // namespace NLog
} // namespace NYql
diff --git a/ydb/library/yql/utils/log/log.cpp b/ydb/library/yql/utils/log/log.cpp
index b62bfeb907..cfb8c4521a 100644
--- a/ydb/library/yql/utils/log/log.cpp
+++ b/ydb/library/yql/utils/log/log.cpp
@@ -1,16 +1,16 @@
-#include "log.h"
-
+#include "log.h"
+
#include <library/cpp/logger/stream.h>
#include <library/cpp/logger/system.h>
-
+
#include <util/datetime/systime.h>
#include <util/generic/strbuf.h>
#include <util/stream/format.h>
#include <util/system/getpid.h>
#include <util/system/mutex.h>
-#include <util/system/progname.h>
+#include <util/system/progname.h>
#include <util/system/thread.i>
-
+
#include <stdio.h>
#include <time.h>
@@ -35,44 +35,44 @@ void WriteLocalTime(IOutputStream* out) {
}
class TLimitedLogBackend final : public TLogBackend {
-public:
- TLimitedLogBackend(TAutoPtr<TLogBackend> b, TAtomic& flag, ui64 limit) noexcept
- : Backend(b)
- , Flag(flag)
- , Limit(limit)
- {
- }
-
- ~TLimitedLogBackend() final {
- }
-
- void ReopenLog() final {
- Backend->ReopenLog();
- }
-
- void WriteData(const TLogRecord& rec) final {
- const auto remaining = AtomicGet(Limit);
- const bool final = remaining > 0 && AtomicSub(Limit, rec.Len) <= 0;
- if (remaining > 0 || rec.Priority <= TLOG_WARNING) {
- Backend->WriteData(rec);
- }
- if (final) {
- AtomicSet(Flag, 1);
- }
- }
-
-private:
- THolder<TLogBackend> Backend;
- TAtomic& Flag;
- TAtomic Limit;
-};
-
-
+public:
+ TLimitedLogBackend(TAutoPtr<TLogBackend> b, TAtomic& flag, ui64 limit) noexcept
+ : Backend(b)
+ , Flag(flag)
+ , Limit(limit)
+ {
+ }
+
+ ~TLimitedLogBackend() final {
+ }
+
+ void ReopenLog() final {
+ Backend->ReopenLog();
+ }
+
+ void WriteData(const TLogRecord& rec) final {
+ const auto remaining = AtomicGet(Limit);
+ const bool final = remaining > 0 && AtomicSub(Limit, rec.Len) <= 0;
+ if (remaining > 0 || rec.Priority <= TLOG_WARNING) {
+ Backend->WriteData(rec);
+ }
+ if (final) {
+ AtomicSet(Flag, 1);
+ }
+ }
+
+private:
+ THolder<TLogBackend> Backend;
+ TAtomic& Flag;
+ TAtomic Limit;
+};
+
+
} // namspace
-namespace NYql {
+namespace NYql {
namespace NLog {
-
+
/**
* TYqlLogElement
* automaticaly adds new line char
@@ -99,7 +99,7 @@ TYqlLog::TYqlLog(const TString& logType, const TComponentLevels& levels)
: TLog(logType)
, ProcName_(GetProgramName())
, ProcId_(GetPID())
- , WriteTruncMsg_(0)
+ , WriteTruncMsg_(0)
{
for (size_t component = 0; component < levels.size(); ++component) {
SetComponentLevel(EComponentHelpers::FromInt(component), levels[component]);
@@ -110,7 +110,7 @@ TYqlLog::TYqlLog(TAutoPtr<TLogBackend> backend, const TComponentLevels& levels)
: TLog(backend)
, ProcName_(GetProgramName())
, ProcId_(GetPID())
- , WriteTruncMsg_(0)
+ , WriteTruncMsg_(0)
{
for (size_t component = 0; component < levels.size(); ++component) {
SetComponentLevel(EComponentHelpers::FromInt(component), levels[component]);
@@ -124,27 +124,27 @@ void TYqlLog::UpdateProcInfo(const TString& procName) {
TAutoPtr<TLogElement> TYqlLog::CreateLogElement(
EComponent component, ELevel level,
- TStringBuf file, int line) const
+ TStringBuf file, int line) const
{
- const bool writeMsg = AtomicCas(&WriteTruncMsg_, 0, 1);
- auto element = MakeHolder<TYqlLogElement>(this, writeMsg ? ELevel::FATAL : level);
- if (writeMsg) {
- WriteLogPrefix(element.Get(), EComponent::Default, ELevel::FATAL, __FILE__, __LINE__);
- *element << "Log is truncated by limit\n";
- *element << ELevelHelpers::ToLogPriority(level);
- }
-
- WriteLogPrefix(element.Get(), component, level, file, line);
- return element.Release();
-}
-
-void TYqlLog::WriteLogPrefix(IOutputStream* out, EComponent component, ELevel level, TStringBuf file, int line) const {
+ const bool writeMsg = AtomicCas(&WriteTruncMsg_, 0, 1);
+ auto element = MakeHolder<TYqlLogElement>(this, writeMsg ? ELevel::FATAL : level);
+ if (writeMsg) {
+ WriteLogPrefix(element.Get(), EComponent::Default, ELevel::FATAL, __FILE__, __LINE__);
+ *element << "Log is truncated by limit\n";
+ *element << ELevelHelpers::ToLogPriority(level);
+ }
+
+ WriteLogPrefix(element.Get(), component, level, file, line);
+ return element.Release();
+}
+
+void TYqlLog::WriteLogPrefix(IOutputStream* out, EComponent component, ELevel level, TStringBuf file, int line) const {
// LOG FORMAT:
// {datetime} {level} {procname}(pid={pid}, tid={tid}) [{component}] {source_location}: {message}\n
//
- WriteLocalTime(out);
- *out << ' '
+ WriteLocalTime(out);
+ *out << ' '
<< ELevelHelpers::ToString(level) << ' '
<< ProcName_ << TStringBuf("(pid=") << ProcId_
<< TStringBuf(", tid=")
@@ -156,10 +156,10 @@ void TYqlLog::WriteLogPrefix(IOutputStream* out, EComponent component, ELevel le
<< TStringBuf(") [") << EComponentHelpers::ToString(component)
<< TStringBuf("] ")
<< file.RAfter(LOCSLASH_C) << ':' << line << TStringBuf(": ");
-}
+}
-void TYqlLog::SetMaxLogLimit(ui64 limit) {
- auto backend = TLog::ReleaseBackend();
+void TYqlLog::SetMaxLogLimit(ui64 limit) {
+ auto backend = TLog::ReleaseBackend();
TLog::ResetBackend(THolder(new TLimitedLogBackend(backend, WriteTruncMsg_, limit)));
}
@@ -190,9 +190,9 @@ void InitLogger(const TString& logType, bool startAsDaemon) {
} else {
TLoggerOperator<TYqlLog>::Set(new TYqlLog(logType, levels));
}
- }
-}
-
+ }
+}
+
void InitLogger(TAutoPtr<TLogBackend> backend) {
with_lock(g_InitLoggerMutex) {
++g_LoggerInitialized;
@@ -222,7 +222,7 @@ void CleanupLogger() {
}
} // namespace NLog
-} // namespace NYql
+} // namespace NYql
/**
* creates default YQL logger writing to /dev/null
diff --git a/ydb/library/yql/utils/log/log.h b/ydb/library/yql/utils/log/log.h
index 8095fdd495..17fcd71627 100644
--- a/ydb/library/yql/utils/log/log.h
+++ b/ydb/library/yql/utils/log/log.h
@@ -1,16 +1,16 @@
-#pragma once
-
+#pragma once
+
#include "log_component.h"
#include "log_level.h"
#include "context.h"
#include "profile.h"
-
+
#include <library/cpp/logger/global/common.h>
-#include <util/system/atomic.h>
-#include <util/stream/output.h>
-#include <util/generic/strbuf.h>
-
+#include <util/system/atomic.h>
+#include <util/stream/output.h>
+#include <util/generic/strbuf.h>
+
#include <array>
@@ -84,12 +84,12 @@
YQL_LOG_PREP_IF(level, ::NYql::NLog::TContextPreprocessor, condition)
-namespace NYql {
+namespace NYql {
namespace NLog {
-
+
using TComponentLevels =
std::array<ELevel, EComponentHelpers::ToInt(EComponent::MaxValue)>;
-
+
/**
* @brief Component based logger frontend.
*/
@@ -114,17 +114,17 @@ public:
return ELevelHelpers::Lte(level, GetComponentLevel(component));
}
- void SetMaxLogLimit(ui64 limit);
-
- TAutoPtr<TLogElement> CreateLogElement(EComponent component, ELevel level, TStringBuf file, int line) const;
-
- void WriteLogPrefix(IOutputStream* out, EComponent component, ELevel level, TStringBuf file, int line) const;
+ void SetMaxLogLimit(ui64 limit);
-private:
+ TAutoPtr<TLogElement> CreateLogElement(EComponent component, ELevel level, TStringBuf file, int line) const;
+
+ void WriteLogPrefix(IOutputStream* out, EComponent component, ELevel level, TStringBuf file, int line) const;
+
+private:
TString ProcName_;
pid_t ProcId_;
std::array<TAtomic, EComponentHelpers::ToInt(EComponent::MaxValue)> ComponentLevels_{0};
- mutable TAtomic WriteTruncMsg_;
+ mutable TAtomic WriteTruncMsg_;
};
/**
@@ -175,7 +175,7 @@ public:
};
} // namespace NLog
-} // namespace NYql
+} // namespace NYql
template <>
NYql::NLog::TYqlLog* CreateDefaultLogger<NYql::NLog::TYqlLog>();
diff --git a/ydb/library/yql/utils/log/log_component.h b/ydb/library/yql/utils/log/log_component.h
index d7a1e10ca1..e2f2b33334 100644
--- a/ydb/library/yql/utils/log/log_component.h
+++ b/ydb/library/yql/utils/log/log_component.h
@@ -24,9 +24,9 @@ enum class EComponent {
Net,
ProviderStat,
ProviderSolomon,
- CoreEval,
- CorePeepHole,
- ProviderDq,
+ CoreEval,
+ CorePeepHole,
+ ProviderDq,
ProviderClickHouse,
ProviderYdb,
ProviderPq,
diff --git a/ydb/library/yql/utils/log/log_ut.cpp b/ydb/library/yql/utils/log/log_ut.cpp
index bf8b71476c..8f4a7cbad8 100644
--- a/ydb/library/yql/utils/log/log_ut.cpp
+++ b/ydb/library/yql/utils/log/log_ut.cpp
@@ -446,94 +446,94 @@ Y_UNIT_TEST_SUITE(TLogTest)
UNIT_ASSERT_C(isMatch, "Unexpected message: " << logRow.Message);
}
}
-
- Y_UNIT_TEST(Limit1) {
- size_t limit = 0;
- {
- TStringStream out;
+
+ Y_UNIT_TEST(Limit1) {
+ size_t limit = 0;
+ {
+ TStringStream out;
YqlLoggerScope logger(&out);
- YqlLogger().UpdateProcInfo("proc");
- YQL_CLOG(INFO, Core) << "message1";
- limit = out.Str().length() * 2 - 7; // Not more than 2 log lines
- }
-
- TStringStream out;
+ YqlLogger().UpdateProcInfo("proc");
+ YQL_CLOG(INFO, Core) << "message1";
+ limit = out.Str().length() * 2 - 7; // Not more than 2 log lines
+ }
+
+ TStringStream out;
YqlLoggerScope logger(&out);
- YqlLogger().UpdateProcInfo("proc");
- YqlLogger().SetMaxLogLimit(limit);
-
- YQL_CLOG(INFO, Core) << "message1";
- YQL_CLOG(INFO, Core) << "message2";
- YQL_CLOG(INFO, Core) << "message3";
-
- TString row1Str, row2Str, row3Str, _;
- Split(out.Str(), '\n', row1Str, row2Str, row3Str, _);
-
- {
- TLogRow logRow = ParseLogRow(row1Str);
- UNIT_ASSERT_EQUAL(logRow.Level, ELevel::INFO);
- UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Core);
- UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "message1");
- }
- {
- TLogRow logRow = ParseLogRow(row2Str);
- UNIT_ASSERT_EQUAL(logRow.Level, ELevel::INFO);
- UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Core);
- UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "message2");
- }
- {
- TLogRow logRow = ParseLogRow(row3Str);
- UNIT_ASSERT_EQUAL(logRow.Level, ELevel::FATAL);
- UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Default);
- UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "Log is truncated by limit");
- }
- }
-
- Y_UNIT_TEST(Limit2) {
- size_t limit = 0;
- {
- TStringStream out;
+ YqlLogger().UpdateProcInfo("proc");
+ YqlLogger().SetMaxLogLimit(limit);
+
+ YQL_CLOG(INFO, Core) << "message1";
+ YQL_CLOG(INFO, Core) << "message2";
+ YQL_CLOG(INFO, Core) << "message3";
+
+ TString row1Str, row2Str, row3Str, _;
+ Split(out.Str(), '\n', row1Str, row2Str, row3Str, _);
+
+ {
+ TLogRow logRow = ParseLogRow(row1Str);
+ UNIT_ASSERT_EQUAL(logRow.Level, ELevel::INFO);
+ UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Core);
+ UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "message1");
+ }
+ {
+ TLogRow logRow = ParseLogRow(row2Str);
+ UNIT_ASSERT_EQUAL(logRow.Level, ELevel::INFO);
+ UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Core);
+ UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "message2");
+ }
+ {
+ TLogRow logRow = ParseLogRow(row3Str);
+ UNIT_ASSERT_EQUAL(logRow.Level, ELevel::FATAL);
+ UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Default);
+ UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "Log is truncated by limit");
+ }
+ }
+
+ Y_UNIT_TEST(Limit2) {
+ size_t limit = 0;
+ {
+ TStringStream out;
YqlLoggerScope logger(&out);
- YqlLogger().UpdateProcInfo("proc");
- YQL_CLOG(INFO, Core) << "message1";
- limit = out.Str().length() * 2 - 7; // Not more than 2 log lines
- }
-
- TStringStream out;
+ YqlLogger().UpdateProcInfo("proc");
+ YQL_CLOG(INFO, Core) << "message1";
+ limit = out.Str().length() * 2 - 7; // Not more than 2 log lines
+ }
+
+ TStringStream out;
YqlLoggerScope logger(&out);
- YqlLogger().UpdateProcInfo("proc");
- YqlLogger().SetMaxLogLimit(limit);
-
- YQL_CLOG(INFO, Core) << "message1";
- YQL_CLOG(INFO, Core) << "message2";
- YQL_CLOG(WARN, Core) << "message3";
-
- TString row1Str, row2Str, row3Str, row4Str, _;
- Split(out.Str(), '\n', row1Str, row2Str, row3Str, row4Str, _);
-
- {
- TLogRow logRow = ParseLogRow(row1Str);
- UNIT_ASSERT_EQUAL(logRow.Level, ELevel::INFO);
- UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Core);
- UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "message1");
- }
- {
- TLogRow logRow = ParseLogRow(row2Str);
- UNIT_ASSERT_EQUAL(logRow.Level, ELevel::INFO);
- UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Core);
- UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "message2");
- }
- {
- TLogRow logRow = ParseLogRow(row3Str);
- UNIT_ASSERT_EQUAL(logRow.Level, ELevel::FATAL);
- UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Default);
- UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "Log is truncated by limit");
- }
- {
- TLogRow logRow = ParseLogRow(row4Str);
- UNIT_ASSERT_EQUAL(logRow.Level, ELevel::WARN);
- UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Core);
- UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "message3");
- }
- }
+ YqlLogger().UpdateProcInfo("proc");
+ YqlLogger().SetMaxLogLimit(limit);
+
+ YQL_CLOG(INFO, Core) << "message1";
+ YQL_CLOG(INFO, Core) << "message2";
+ YQL_CLOG(WARN, Core) << "message3";
+
+ TString row1Str, row2Str, row3Str, row4Str, _;
+ Split(out.Str(), '\n', row1Str, row2Str, row3Str, row4Str, _);
+
+ {
+ TLogRow logRow = ParseLogRow(row1Str);
+ UNIT_ASSERT_EQUAL(logRow.Level, ELevel::INFO);
+ UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Core);
+ UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "message1");
+ }
+ {
+ TLogRow logRow = ParseLogRow(row2Str);
+ UNIT_ASSERT_EQUAL(logRow.Level, ELevel::INFO);
+ UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Core);
+ UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "message2");
+ }
+ {
+ TLogRow logRow = ParseLogRow(row3Str);
+ UNIT_ASSERT_EQUAL(logRow.Level, ELevel::FATAL);
+ UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Default);
+ UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "Log is truncated by limit");
+ }
+ {
+ TLogRow logRow = ParseLogRow(row4Str);
+ UNIT_ASSERT_EQUAL(logRow.Level, ELevel::WARN);
+ UNIT_ASSERT_EQUAL(logRow.Component, EComponent::Core);
+ UNIT_ASSERT_STRINGS_EQUAL(logRow.Message, "message3");
+ }
+ }
}
diff --git a/ydb/library/yql/utils/log/ya.make b/ydb/library/yql/utils/log/ya.make
index e83464fe9f..6836d8a9de 100644
--- a/ydb/library/yql/utils/log/ya.make
+++ b/ydb/library/yql/utils/log/ya.make
@@ -1,23 +1,23 @@
-LIBRARY()
-
+LIBRARY()
+
OWNER(
udovichenko-r
g:yql
)
-
-SRCS(
+
+SRCS(
context.cpp
- log.cpp
+ log.cpp
profile.cpp
tls_backend.cpp
-)
-
-PEERDIR(
+)
+
+PEERDIR(
library/cpp/logger
library/cpp/logger/global
-)
-
-END()
+)
+
+END()
RECURSE_FOR_TESTS(
ut
diff --git a/ydb/library/yql/utils/proc_alive.cpp b/ydb/library/yql/utils/proc_alive.cpp
index 7efb2584bc..ce5a879964 100644
--- a/ydb/library/yql/utils/proc_alive.cpp
+++ b/ydb/library/yql/utils/proc_alive.cpp
@@ -1,37 +1,37 @@
-#include "proc_alive.h"
-
-#include <util/system/platform.h>
-#include <util/system/compat.h>
-#include <util/system/error.h>
-#include <util/system/winint.h>
-
-#include <errno.h>
-
-
-namespace NYql {
-
-bool IsProcessAlive(TProcessId pid) {
-#ifdef _unix_
- // If sending a null signal fails with the error ESRCH, then we know
- // the process doesn’t exist. If the call fails with the error
- // EPERM - the process exists, but we don’t have permission to send
- // a signal to it.
- kill(pid, 0);
- return errno != ESRCH;
-#elif defined(_win_)
- HANDLE process = OpenProcess(SYNCHRONIZE, FALSE, pid);
- if (process == NULL) {
- return false;
- }
-
- DWORD ret = WaitForSingleObject(process, 0);
- CloseHandle(process);
- return ret == WAIT_TIMEOUT;
-#else
- Y_UNUSED(pid);
- return false;
-#endif
-}
-
-} // NYql
-
+#include "proc_alive.h"
+
+#include <util/system/platform.h>
+#include <util/system/compat.h>
+#include <util/system/error.h>
+#include <util/system/winint.h>
+
+#include <errno.h>
+
+
+namespace NYql {
+
+bool IsProcessAlive(TProcessId pid) {
+#ifdef _unix_
+ // If sending a null signal fails with the error ESRCH, then we know
+ // the process doesn’t exist. If the call fails with the error
+ // EPERM - the process exists, but we don’t have permission to send
+ // a signal to it.
+ kill(pid, 0);
+ return errno != ESRCH;
+#elif defined(_win_)
+ HANDLE process = OpenProcess(SYNCHRONIZE, FALSE, pid);
+ if (process == NULL) {
+ return false;
+ }
+
+ DWORD ret = WaitForSingleObject(process, 0);
+ CloseHandle(process);
+ return ret == WAIT_TIMEOUT;
+#else
+ Y_UNUSED(pid);
+ return false;
+#endif
+}
+
+} // NYql
+
diff --git a/ydb/library/yql/utils/proc_alive.h b/ydb/library/yql/utils/proc_alive.h
index c4b798b4ca..55cb53f0e1 100644
--- a/ydb/library/yql/utils/proc_alive.h
+++ b/ydb/library/yql/utils/proc_alive.h
@@ -1,9 +1,9 @@
-#pragma once
-
-#include <util/system/getpid.h>
-
-namespace NYql {
-
-bool IsProcessAlive(TProcessId pid);
-
-}
+#pragma once
+
+#include <util/system/getpid.h>
+
+namespace NYql {
+
+bool IsProcessAlive(TProcessId pid);
+
+}
diff --git a/ydb/library/yql/utils/threading/async_semaphore.cpp b/ydb/library/yql/utils/threading/async_semaphore.cpp
index 9390fa3ce6..f0ffd3b124 100644
--- a/ydb/library/yql/utils/threading/async_semaphore.cpp
+++ b/ydb/library/yql/utils/threading/async_semaphore.cpp
@@ -1,75 +1,75 @@
-#include "async_semaphore.h"
-
-#include <util/system/guard.h>
-#include <util/system/yassert.h>
-
-namespace NYql {
-
-TAsyncSemaphore::TAsyncSemaphore(size_t count)
- : Count_(count)
-{
- Y_ASSERT(count > 0);
-}
-
-TAsyncSemaphore::TPtr TAsyncSemaphore::Make(size_t count) {
- return TPtr(new TAsyncSemaphore(count));
-}
-
-NThreading::TFuture<TAsyncSemaphore::TPtr> TAsyncSemaphore::AcquireAsync() {
- with_lock(Lock_) {
- if (Count_) {
- --Count_;
- return NThreading::MakeFuture<TAsyncSemaphore::TPtr>(this);
- }
- auto promise = NThreading::NewPromise<TAsyncSemaphore::TPtr>();
- Promises_.push_back(promise);
- return promise.GetFuture();
- }
-}
-
-void TAsyncSemaphore::Release() {
- NThreading::TPromise<TPtr> promise;
- with_lock(Lock_) {
- if (Promises_.empty()) {
- ++Count_;
- return;
- } else {
- promise = Promises_.front();
- Promises_.pop_front();
- }
- }
- promise.SetValue(this);
-}
-
-void TAsyncSemaphore::Cancel() {
- std::list<NThreading::TPromise<TPtr>> promises;
- with_lock(Lock_) {
- std::swap(Promises_, promises);
- }
- for (auto& p: promises) {
- p.SetException("Cancelled");
- }
-}
-
-TAsyncSemaphore::TAutoRelease::~TAutoRelease() {
- if (Sem) {
- Sem->Release();
- }
-}
-
-std::function<void (const NThreading::TFuture<void>&)> TAsyncSemaphore::TAutoRelease::DeferRelease() {
- return [s = std::move(this->Sem)](const NThreading::TFuture<void>& f) {
- f.GetValue();
- s->Release();
- };
-}
-
-
-std::function<void (const NThreading::TFuture<void>&)> TAsyncSemaphore::DeferRelease() {
- return [s = TPtr(this)](const NThreading::TFuture<void>& f) {
- f.GetValue();
- s->Release();
- };
-}
-
-}
+#include "async_semaphore.h"
+
+#include <util/system/guard.h>
+#include <util/system/yassert.h>
+
+namespace NYql {
+
+TAsyncSemaphore::TAsyncSemaphore(size_t count)
+ : Count_(count)
+{
+ Y_ASSERT(count > 0);
+}
+
+TAsyncSemaphore::TPtr TAsyncSemaphore::Make(size_t count) {
+ return TPtr(new TAsyncSemaphore(count));
+}
+
+NThreading::TFuture<TAsyncSemaphore::TPtr> TAsyncSemaphore::AcquireAsync() {
+ with_lock(Lock_) {
+ if (Count_) {
+ --Count_;
+ return NThreading::MakeFuture<TAsyncSemaphore::TPtr>(this);
+ }
+ auto promise = NThreading::NewPromise<TAsyncSemaphore::TPtr>();
+ Promises_.push_back(promise);
+ return promise.GetFuture();
+ }
+}
+
+void TAsyncSemaphore::Release() {
+ NThreading::TPromise<TPtr> promise;
+ with_lock(Lock_) {
+ if (Promises_.empty()) {
+ ++Count_;
+ return;
+ } else {
+ promise = Promises_.front();
+ Promises_.pop_front();
+ }
+ }
+ promise.SetValue(this);
+}
+
+void TAsyncSemaphore::Cancel() {
+ std::list<NThreading::TPromise<TPtr>> promises;
+ with_lock(Lock_) {
+ std::swap(Promises_, promises);
+ }
+ for (auto& p: promises) {
+ p.SetException("Cancelled");
+ }
+}
+
+TAsyncSemaphore::TAutoRelease::~TAutoRelease() {
+ if (Sem) {
+ Sem->Release();
+ }
+}
+
+std::function<void (const NThreading::TFuture<void>&)> TAsyncSemaphore::TAutoRelease::DeferRelease() {
+ return [s = std::move(this->Sem)](const NThreading::TFuture<void>& f) {
+ f.GetValue();
+ s->Release();
+ };
+}
+
+
+std::function<void (const NThreading::TFuture<void>&)> TAsyncSemaphore::DeferRelease() {
+ return [s = TPtr(this)](const NThreading::TFuture<void>& f) {
+ f.GetValue();
+ s->Release();
+ };
+}
+
+}
diff --git a/ydb/library/yql/utils/threading/async_semaphore.h b/ydb/library/yql/utils/threading/async_semaphore.h
index f72fc64e17..e074db010f 100644
--- a/ydb/library/yql/utils/threading/async_semaphore.h
+++ b/ydb/library/yql/utils/threading/async_semaphore.h
@@ -1,55 +1,55 @@
-#pragma once
-
-#include <library/cpp/threading/future/future.h>
-
-#include <util/system/spinlock.h>
-#include <util/generic/ptr.h>
-
-#include <list>
-#include <functional>
-
-namespace NYql {
-
-class TAsyncSemaphore: public TThrRefBase {
-public:
- using TPtr = TIntrusivePtr<TAsyncSemaphore>;
-
- class TAutoRelease {
- public:
- TAutoRelease(TAsyncSemaphore::TPtr sem)
- : Sem(std::move(sem))
- {
- }
- TAutoRelease(TAutoRelease&& other)
- : Sem(std::move(other.Sem))
- {
- }
- ~TAutoRelease();
-
- std::function<void (const NThreading::TFuture<void>&)> DeferRelease();
-
- private:
- TAsyncSemaphore::TPtr Sem;
- };
-
- static TPtr Make(size_t count);
-
- NThreading::TFuture<TPtr> AcquireAsync();
- void Release();
- void Cancel();
- std::function<void (const NThreading::TFuture<void>&)> DeferRelease();
-
- TAutoRelease MakeAutoRelease() {
- return {this};
- }
-
-private:
- TAsyncSemaphore(size_t count);
-
-private:
- size_t Count_;
- TAdaptiveLock Lock_;
- std::list<NThreading::TPromise<TPtr>> Promises_;
-};
-
-} // NYql
+#pragma once
+
+#include <library/cpp/threading/future/future.h>
+
+#include <util/system/spinlock.h>
+#include <util/generic/ptr.h>
+
+#include <list>
+#include <functional>
+
+namespace NYql {
+
+class TAsyncSemaphore: public TThrRefBase {
+public:
+ using TPtr = TIntrusivePtr<TAsyncSemaphore>;
+
+ class TAutoRelease {
+ public:
+ TAutoRelease(TAsyncSemaphore::TPtr sem)
+ : Sem(std::move(sem))
+ {
+ }
+ TAutoRelease(TAutoRelease&& other)
+ : Sem(std::move(other.Sem))
+ {
+ }
+ ~TAutoRelease();
+
+ std::function<void (const NThreading::TFuture<void>&)> DeferRelease();
+
+ private:
+ TAsyncSemaphore::TPtr Sem;
+ };
+
+ static TPtr Make(size_t count);
+
+ NThreading::TFuture<TPtr> AcquireAsync();
+ void Release();
+ void Cancel();
+ std::function<void (const NThreading::TFuture<void>&)> DeferRelease();
+
+ TAutoRelease MakeAutoRelease() {
+ return {this};
+ }
+
+private:
+ TAsyncSemaphore(size_t count);
+
+private:
+ size_t Count_;
+ TAdaptiveLock Lock_;
+ std::list<NThreading::TPromise<TPtr>> Promises_;
+};
+
+} // NYql
diff --git a/ydb/library/yql/utils/threading/ya.make b/ydb/library/yql/utils/threading/ya.make
index 3a4ed1abaa..1d80c91a2a 100644
--- a/ydb/library/yql/utils/threading/ya.make
+++ b/ydb/library/yql/utils/threading/ya.make
@@ -1,9 +1,9 @@
-LIBRARY()
-
-OWNER(g:yql)
-
-SRCS(
- async_semaphore.cpp
-)
-
-END()
+LIBRARY()
+
+OWNER(g:yql)
+
+SRCS(
+ async_semaphore.cpp
+)
+
+END()
diff --git a/ydb/library/yql/utils/ya.make b/ydb/library/yql/utils/ya.make
index afba70be53..c7eac08e88 100644
--- a/ydb/library/yql/utils/ya.make
+++ b/ydb/library/yql/utils/ya.make
@@ -21,8 +21,8 @@ SRCS(
multi_resource_lock.h
parse_double.cpp
parse_double.h
- proc_alive.cpp
- proc_alive.h
+ proc_alive.cpp
+ proc_alive.h
rand_guid.cpp
rand_guid.h
resetable_setting.h